Added library for scalling jenkins

Change-Id: I49932f43ac4e41f06c559d1d18d142b8a084278e
Signed-off-by: vamahaja <vamahaja@redhat.com>

15 files changed, 890 insertions, 0 deletions

diff --git a/cns-libs/cnslibs/cns/cns_baseclass.py b/cns-libs/cnslibs/cns/cns_baseclass.py
index 2cc1abf1..bffc4122 100644
--- a/cns-libs/cnslibs/cns/cns_baseclass.py
+++ b/cns-libs/cnslibs/cns/cns_baseclass.py
@@ -349,3 +349,18 @@ class CnsGlusterBlockBaseClass(CnsBaseClass):
             if ret != 0:
                 raise ExecutionError("failed to execute cmd %s on %s out: "
                                      "%s err: %s" % (cmd, node, out, err))
+
+
+class PodScalabilityBaseClass(CnsBaseClass):
+    """
+    This class is for setting parameters for scalling pods
+    """
+    @classmethod
+    def setUpClass(cls):
+        """
+        Initialize all the variables necessary for scalling setup
+        """
+
+        super(PodScalabilityBaseClass, cls).setUpClass()
+
+        cls.scale_info = g.config['scale']
diff --git a/cns-libs/cnslibs/common/dynamic_provisioning.py b/cns-libs/cnslibs/common/dynamic_provisioning.py
index 70dffa4f..7580eb77 100644
--- a/cns-libs/cnslibs/common/dynamic_provisioning.py
+++ b/cns-libs/cnslibs/common/dynamic_provisioning.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from string import Template
 import json
 import os
 import tempfile
@@ -14,6 +15,41 @@ from cnslibs.common.waiter import Waiter
 TEMPLATE_DIR = os.path.abspath(os.path.dirname(__file__))
 
 
+def create_script_template_file(hostname, template_file_path, src_dir_path,
+                                dest_dir_path, params):
+    """Update the templates with the parameters and upload to openshift
+       master node
+    Args:
+        template_file_path (str): template file which need to be updated
+        src_dir_path (str): source path of template file
+        dest_dir_path (str): directory path to which template file
+                                to be generated
+        params (dict): key, val pair to update template file
+    Raises:
+        Raises AssertionError in case of failure
+    """
+
+    src_path = os.path.join(src_dir_path, template_file_path)
+    contents = Template(open(src_path).read())
+    contents = contents.safe_substitute(params)
+
+    dest_path = os.path.join(dest_dir_path, template_file_path)
+    try:
+        conn = g.rpyc_get_connection(hostname, user="root")
+        if conn is None:
+            g.log.error("Failed to get rpyc connection of node %s"
+                        % hostname)
+            raise AssertionError("Failed to get rpyc connection of node %s"
+                                 % hostname)
+        with conn.builtin.open(dest_path, 'w') as data_file:
+            data_file.write(contents)
+    except Exception as err:
+        g.log.error("failed to create script file %s" % err)
+        raise
+    finally:
+        g.rpyc_close_connection(hostname, user="root")
+
+
 def create_pvc_file(hostname, claim_name, storage_class, size):
     '''
      This function creates pvc file
diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py
index ea1cea6a..3cba8e00 100644
--- a/cns-libs/cnslibs/common/openshift_ops.py
+++ b/cns-libs/cnslibs/common/openshift_ops.py
@@ -152,6 +152,28 @@ def switch_oc_project(ocp_node, project_name):
     return True
 
 
+def oc_rsync(ocp_node, pod_name, src_dir_path, dest_dir_path):
+    """Sync file from 'src_dir_path' path on ocp_node to
+       'dest_dir_path' path on 'pod_name' using 'oc rsync' command.
+
+    Args:
+        ocp_node (str): Node on which oc rsync command will be executed
+        pod_name (str): Name of the pod on which source directory to be
+                        mounted
+        src_dir_path (path): Source path from which directory to be mounted
+        dest_dir_path (path): destination path to which directory to be
+                              mounted
+    """
+    ret, out, err = g.run(ocp_node, ['oc',
+                                     'rsync',
+                                     src_dir_path,
+                                     '%s:%s' % (pod_name, dest_dir_path)])
+    if ret != 0:
+        error_msg = 'failed to sync directory in pod: %r; %r' % (out, err)
+        g.log.error(error_msg)
+        raise AssertionError(error_msg)
+
+
 def oc_rsh(ocp_node, pod_name, command, log_level=None):
     """Run a command in the ocp pod using `oc rsh`.
 
@@ -201,6 +223,28 @@ def oc_create(ocp_node, value, value_type='file'):
     g.log.info('Created resource from %s.' % value_type)
 
 
+def oc_process(ocp_node, params, filename):
+    """Create a resource template based on the contents of the
+       given filename and params provided.
+    Args:
+        ocp_node (str): Node on which the ocp command will run
+        filename (str): Filename (on remote) to be passed to
+                        oc process command.
+    Returns: template generated through process command
+    Raises:
+        AssertionError: Raised when resource fails to create.
+    """
+
+    ret, out, err = g.run(ocp_node, ['oc', 'process', '-f', filename, params])
+    if ret != 0:
+        error_msg = 'failed to create process: %r; %r' % (out, err)
+        g.log.error(error_msg)
+        raise AssertionError(error_msg)
+    g.log.info('Created resource from file (%s)', filename)
+
+    return out
+
+
 def oc_create_secret(hostname, secret_name_prefix="autotests-secret-",
                      namespace="default",
                      data_key="password",
diff --git a/cns-libs/cnslibs/common/pod_libs.py b/cns-libs/cnslibs/common/pod_libs.py
new file mode 100644
index 00000000..092a6f84
--- /dev/null
+++ b/cns-libs/cnslibs/common/pod_libs.py
@@ -0,0 +1,342 @@
+
+import os
+import tempfile
+import yaml
+
+from shutil import copytree
+from glusto.core import Glusto as g
+from glustolibs.misc.misc_libs import upload_scripts
+
+from openshift_ops import (
+    oc_process,
+    oc_create,
+    oc_rsh,
+    oc_rsync,
+    oc_get_yaml,
+    create_namespace,
+    switch_oc_project
+)
+
+from dynamic_provisioning import (
+    wait_for_pod_be_ready,
+    get_pod_name_from_dc
+)
+
+
+CNS_TEMPLATE_PATH = os.path.join(
+    os.getcwdu(),
+    "cns-libs",
+    "templates"
+)
+ROOT = os.sep
+
+LOAD_TYPE_JENKINS = "jenkins"
+JENKINS_TEMPLATE = "jenkins-persistent-template.yaml"
+JJB_TEMPLATE = "jjb-template.yaml"
+JENKINS_LOAD_SCRIPT = "start-load-on-jenkins.j2"
+JENKINS_GROOVY_SCRIPTS = [
+    "install_mvn339.groovy",
+    "install_jdk8.groovy",
+    "jenkins_setup_jdk_user.groovy.j2"
+]
+
+
+def deploy_pods(oc_master_node, load_type, instances, pod_parameters,
+                load_parameters, namespace=None):
+    """Deploy and configure pods of different types
+    Args:
+        load_type (str): load type jenkins, mongodb or postgres
+        instances (int): number of pods to be deployed
+        pod_parameters (str): pod parameters used to deploy pods
+        load_parameters (str): load parameters to be used to start
+                               load on pods
+        namespace (str): namespace used to deploy jenkins and jjb pods
+    Raises: AssertionError if invalid load type specified
+    """
+    err_msg = "invalid load typed %s"
+
+    if namespace:
+        create_and_switch_to_namespace(oc_master_node, namespace)
+
+    templates_path = upload_templates(oc_master_node, load_type)
+
+    pod_index = 0
+    for index in range(int(instances)):
+        if load_type == LOAD_TYPE_JENKINS:
+            pod_index = get_available_pod_index(
+                oc_master_node,
+                pod_index,
+                LOAD_TYPE_JENKINS
+            )
+            deploy_and_configure_jenkins(
+                oc_master_node,
+                pod_index,
+                templates_path,
+                pod_parameters,
+                load_parameters
+            )
+        else:
+            g.log.error(err_msg % load_type)
+            raise AssertionError(err_msg % load_type)
+
+
+def upload_templates(oc_master_node, load_type):
+    """Upload templates in openshift master node in temporary
+       directory.
+    Args:
+        load_type (str): load type of which template files need to be uploaded
+    Return:
+        directory path: Return temporary directory path where templates are
+                        copied.
+    Raises: AssertionError if failed to upload files on server
+    """
+    err_msg = "failed to %s templates from %s to %s"
+
+    temp_dir = tempfile.mkdtemp(prefix="cns-tmp-")
+    src = os.path.join(CNS_TEMPLATE_PATH, load_type)
+    dst = os.path.join(temp_dir, load_type)
+    try:
+        copytree(src, dst)
+    except Exception:
+        g.log.error(err_msg % ("copy", src, dst))
+        raise
+
+    dest = os.path.join(ROOT, "tmp")
+    if not upload_scripts(oc_master_node, temp_dir, dest):
+        g.log.error(err_msg % ("upload", temp_dir, dest))
+        raise AssertionError(err_msg % ("upload", temp_dir, dest))
+
+    return os.path.join(temp_dir, load_type)
+
+
+def create_and_switch_to_namespace(oc_master_node, namespace):
+    """Create new namespace if not already exists and switch
+       to new created namespace
+    Args:
+        namespace (str): namespace name to be created
+    Raises: AssertionError if failed to create or switch project to namespace
+    """
+    err_msg = "failed to %s namespace %s"
+
+    try:
+        oc_get_yaml(oc_master_node, "namespace", namespace)
+        g.log.info("%s namespace already present using existing one")
+    except AssertionError:
+        g.log.info("%s namespace not present creating new one")
+        if not create_namespace(oc_master_node, namespace):
+            g.log.error(err_msg % ("create", namespace))
+            raise AssertionError(err_msg % ("create", namespace))
+
+    g.log.info("switch to namespace %s'" % namespace)
+    if not switch_oc_project(oc_master_node, namespace):
+        g.log.error(err_msg % ("switch", namespace))
+        raise AssertionError(err_msg % ("switch", namespace))
+
+
+def create_pod(oc_master_node, templates_path, template, params=None):
+    """Creates pod with template and prameters provides
+    Args:
+        template (str): template by using which pod to be created
+        params (str): parameters used to create template
+    """
+    info_msg = "creating %s using template %s"
+
+    template_path = os.path.join(templates_path, "templates", template)
+    if params:
+        g.log.info(info_msg % ("process", template_path))
+        template = oc_process(oc_master_node, params, template_path)
+
+        g.log.info(info_msg % ("app", template))
+        oc_create(oc_master_node, template, "value")
+    else:
+        g.log.info(info_msg % ("app", template_path))
+        oc_create(oc_master_node, template_path)
+
+
+def get_available_pod_index(oc_master_node, start_index, prefix):
+    """Checks availability of the name with prefix and start_index
+    Args:
+        prefix (str): prefix for the dc name used
+    Returns:
+        start_index (int): maximum no combined with prefix
+    """
+    dc_list = oc_get_yaml(oc_master_node, "dc")
+    dc_names = [dc_name["metadata"]["name"] for dc_name in dc_list["items"]]
+
+    while "%s-%s" % (prefix, start_index) not in dc_names:
+        start_index += 1
+
+    g.log.info("next available index is %s" % start_index)
+    return start_index
+
+
+def deploy_and_configure_jenkins(oc_master_node, index, templates_path,
+                                 pod_parameters, load_parameters):
+    """Deploys and configures jenkins and jjb pod
+    Args:
+        index (int): index no by which pod name need to be appened
+        pod_parameters (str): user specified parameters used to deploy pod
+    """
+    jk_service_name = "%s-%s" % (LOAD_TYPE_JENKINS, index)
+    jnlp_service_name = "jnlp-%s" % index
+    jjb_service_name = "jjb-%s" % index
+
+    deploy_jenkins_pod(
+        oc_master_node,
+        templates_path,
+        jk_service_name,
+        jnlp_service_name,
+        pod_parameters
+    )
+    route_details = oc_get_yaml(
+        oc_master_node,
+        "route",
+        jk_service_name
+    )
+    route = route_details['spec']['host']
+    jk_url = "https://%s" % route
+
+    jjb_pod_name = deploy_jjb_pod(
+        oc_master_node,
+        templates_path,
+        jk_url,
+        jjb_service_name
+    )
+
+    configure_jjb(
+        oc_master_node,
+        load_parameters,
+        templates_path,
+        jjb_pod_name
+    )
+
+    start_jenkins_load(oc_master_node, templates_path, jjb_pod_name, jk_url)
+
+
+def deploy_jenkins_pod(oc_master_node, templates_path, jk_service_name,
+                       jnlp_service_name, pod_parameters):
+    """Deploys jenkins pod with specified parameters
+
+    Args:
+        jk_service_name (str): Jenkins service name to be used for deploying
+                               pod
+        jnlp_service_name (str): JNLP service name to be used for deploying
+                                 pod
+        pod_parameters (str): user specified parameters used to deploy pod
+    Returns:
+        jk_pod_name (str): deployed and ready state Jenkins pod name
+    """
+    params = " ".join(["-p {0}={1}".format(key, val)
+                      for key, val in pod_parameters.items()])
+    params += (" -p JENKINS_SERVICE_NAME=%s -p JNLP_SERVICE_NAME=%s"
+               % (jk_service_name, jnlp_service_name))
+
+    create_pod(oc_master_node, templates_path, JENKINS_TEMPLATE, params)
+    jk_pod_name = get_pod_name_from_dc(oc_master_node, jk_service_name)
+    g.log.info("Jenkins pod %s created successfully" % jk_pod_name)
+
+    g.log.info("waiting pod %s to be in 'Running' state" % jk_pod_name)
+    wait_for_pod_be_ready(oc_master_node, jk_pod_name)
+
+    return jk_pod_name
+
+
+def deploy_jjb_pod(oc_master_node, templates_path, jk_url, jjb_service_name):
+    """Deploys jjb pods with configuring jenkins url jk_url
+
+    Args:
+        jk_url (str): jenkins url to executes groovy scripts and configure
+                      jenkins jobs through jjb
+        jjb_service_name (str) : jjb service used for pod name
+    Returns:
+        jjb_pod_name (str): Deployed and ready state JJB Pod name
+    """
+    params = ("-p JENKINS_URL=%s JJB_SERVICE_NAME=%s"
+              % (jk_url, jjb_service_name))
+    g.log.info("using jenkins url '%s' to deploy jjb pod" % jk_url)
+    create_pod(oc_master_node, templates_path, JJB_TEMPLATE, params)
+
+    jjb_pod_name = get_pod_name_from_dc(oc_master_node, jjb_service_name)
+    g.log.info("deployed jjb pod %s successfully" % jjb_pod_name)
+
+    g.log.info("waiting pod to be in 'Running' state" % jjb_pod_name)
+    wait_for_pod_be_ready(oc_master_node, jjb_pod_name)
+
+    return jjb_pod_name
+
+
+def create_project_template(template_path, load_parameters):
+    """Generates project.yaml file for jjb configuration
+    Args:
+        template_path: Path of template files where project.yml file to be
+                       generated
+        load_parameters: Contains the no of jobs to be generated
+    """
+    project_dict = [{"project": {"name": "svt-jobs", "jobs": []}}]
+
+    for index in range(int(load_parameters["JOBS"])):
+        job = {
+            "{name}_job": {
+                "name": "test-%s" % index,
+                "get_url": "https://github.com/hongkailiu/gs-spring-boot.git"
+            }
+        }
+        project_dict[0]["project"]["jobs"].append(job)
+        project_yaml_path = os.path.join(template_path, "jjb", "project.yaml")
+        with open(project_yaml_path, "w") as f:
+            f.write(yaml.dump(project_dict, default_flow_style=False))
+
+
+def configure_jjb(oc_master_node, load_parameters, templates_path, pod_name):
+    """Copies jjb jobs and load scripts to jjb pod's /data
+        location
+    Args:
+        pod_name (str): jjb pod name on which script to be executed
+    """
+    dest = os.path.join(ROOT, "data")
+
+    create_project_template(templates_path, load_parameters)
+
+    source = os.path.join(templates_path, "jjb", "")
+    g.log.info("copying script files from %s to pod %s" % (source, pod_name))
+    oc_rsync(oc_master_node, pod_name, source, dest)
+
+    source = os.path.join(templates_path, "files", "")
+    g.log.info("copying jenkins load scripts from %s to jenkins pod %s"
+               % (source, pod_name))
+    oc_rsync(oc_master_node, pod_name, source, dest)
+
+
+def start_jenkins_load(oc_master_node, templates_path, pod_name, jk_url):
+    """Configures jenkins settings and start load
+
+    Args:
+        pod_name (str): jjb pod name on which script to be executed
+        jk_url (str): jenkins url to executes groovy scripts and configure
+                      jenkins jobs through jjb
+    Raises: AssertionError if groovy scripts failed to configure jenkins
+    """
+    err_msg = "failed to execute groovy script %s"
+
+    for g_script in JENKINS_GROOVY_SCRIPTS:
+        g.log.info("execute groovy script %s on jenkins" % g_script)
+
+        g_script_path = os.path.join(templates_path, "groovy", g_script)
+        cmd = " ".join([
+            "curl",
+            "-k",
+            "--user", "admin:password",
+            "--data-urlencode",
+            "\"script=$(cat {0})\"".format(g_script_path),
+            "-X", "POST",
+            "{0}/scriptText".format(jk_url)
+        ])
+        ret, out, err = g.run(oc_master_node, cmd)
+        if ret != 0:
+            g.log.error(err_msg % g_script)
+            raise AssertionError(err_msg % g_script)
+
+    g.log.info("start jenkins load script on jjb pod")
+    dest = os.path.join(ROOT, "data")
+    cmd = ["bash", os.path.join(dest, JENKINS_LOAD_SCRIPT), "&"]
+    oc_rsh(oc_master_node, pod_name, cmd)
diff --git a/cns-libs/templates/jenkins/README b/cns-libs/templates/jenkins/README
new file mode 100644
index 00000000..1402a56f
--- /dev/null
+++ b/cns-libs/templates/jenkins/README
@@ -0,0 +1,6 @@
+==============
+REFERENCES
+==============
+
+Jenkins template and load script files are refered from the below github code:
+    https://github.com/openshift/svt/tree/master/storage/jenkins
diff --git a/cns-libs/templates/jenkins/files/start-load-on-jenkins.j2 b/cns-libs/templates/jenkins/files/start-load-on-jenkins.j2
new file mode 100755
index 00000000..7c85e8bd
--- /dev/null
+++ b/cns-libs/templates/jenkins/files/start-load-on-jenkins.j2
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+readonly TOTAL_BUILD_NUMBER=29
+readonly JENKINS_URL=$(awk '/^url/{split ($1, a, "="); print a[2]}' /etc/jenkins_jobs/jenkins_jobs.ini)
+
+echo "JENKINS_URL: $JENKINS_URL"
+
+function trigger() {
+    local url
+    local job_name
+
+    url=$1
+    job_name=$2
+
+    echo "Start job: $job_name"
+    curl -s -k --user admin:password -X POST "${url}/job/${job_name}/build/api/json" --data-urlencode json='{"parameter": []}'
+}
+
+
+function check_build() {
+    local result
+    local job_name
+    local url
+
+    url=$1
+    job_name=$2
+
+    echo "Executing validation for job: ${job_name}"
+    result=$(curl -s -k --user admin:password ${url}/job/${job_name}/lastBuild/api/json | python -c "import json,sys; obj=json.load(sys.stdin); print obj['result'] if 'result' in obj else False")
+
+    echo "Status: ${result}"
+    if [[ "${result}" = "SUCCESS" ]] || [[ "${result}" = "FAILURE" ]] || [[ "${result}" = "UNSTABLE" ]] || [[ "${result}" = "ABORTED" ]]
+    then
+        echo "Build completed, re-starting build"
+        trigger "${JENKINS_URL}" "${job_name}"
+    else
+        echo "Build in progress, waiting for completion"
+    fi
+}
+
+
+### delete jobs
+echo "Delete existing jobs"
+for j in $(seq 0 ${TOTAL_BUILD_NUMBER})
+do
+    jenkins-jobs delete test-${j}_job
+done
+
+echo "Wait for 30 sec until all jobs get created"
+sleep 30
+
+### create jobs
+echo "Create new jobs"
+jenkins-jobs --flush-cache update --delete-old /data
+
+echo "Wait for 30 sec until all jobs get created"
+sleep 30
+
+### trigger jobs
+echo "Start building new job"
+for j in $(seq 0 ${TOTAL_BUILD_NUMBER})
+do
+    trigger "${JENKINS_URL}" "test-${j}_job"
+done
+
+echo "Wait for 2 mins to jenkin schedule and start the jobs"
+sleep 120
+
+### check jobs
+echo "Start polling status for started jobs"
+
+while true
+do
+    for i in $(seq 0 ${TOTAL_BUILD_NUMBER})
+    do
+        check_build "${JENKINS_URL}" "test-${i}_job/job/ttt"
+    done
+
+    echo "Validated all jobs geting some rest"
+    sleep 30
+done
diff --git a/cns-libs/templates/jenkins/groovy/install_jdk8.groovy b/cns-libs/templates/jenkins/groovy/install_jdk8.groovy
new file mode 100644
index 00000000..fb104cdb
--- /dev/null
+++ b/cns-libs/templates/jenkins/groovy/install_jdk8.groovy
@@ -0,0 +1,23 @@
+import jenkins.model.*
+import hudson.model.*
+import hudson.tools.*
+
+def inst = Jenkins.getInstance()
+
+def desc = inst.getDescriptor("hudson.model.JDK")
+
+def versions = [
+  "jdk8": "jdk-8u172-oth-JPR"
+]
+def installations = [];
+
+for (v in versions) {
+  def installer = new JDKInstaller(v.value, true)
+  def installerProps = new InstallSourceProperty([installer])
+  def installation = new JDK(v.key, "", [installerProps])
+  installations.push(installation)
+}
+
+desc.setInstallations(installations.toArray(new JDK[0]))
+
+desc.save()
diff --git a/cns-libs/templates/jenkins/groovy/install_mvn339.groovy b/cns-libs/templates/jenkins/groovy/install_mvn339.groovy
new file mode 100644
index 00000000..4776d240
--- /dev/null
+++ b/cns-libs/templates/jenkins/groovy/install_mvn339.groovy
@@ -0,0 +1,19 @@
+import hudson.tasks.Maven.MavenInstallation;
+import hudson.tools.InstallSourceProperty;
+import hudson.tools.ToolProperty;
+import hudson.tools.ToolPropertyDescriptor;
+import hudson.util.DescribableList;
+
+def mavenDesc = jenkins.model.Jenkins.instance.getExtensionList(hudson.tasks.Maven.DescriptorImpl.class)[0]
+
+def isp = new InstallSourceProperty()
+def autoInstaller = new hudson.tasks.Maven.MavenInstaller("3.3.9")
+isp.installers.add(autoInstaller)
+
+def proplist = new DescribableList<ToolProperty<?>, ToolPropertyDescriptor>()
+proplist.add(isp)
+
+def installation = new MavenInstallation("Maven 3.3.9", "", proplist)
+
+mavenDesc.setInstallations(installation)
+mavenDesc.save()
diff --git a/cns-libs/templates/jenkins/groovy/jenkins_set_no_of_executor.groovy.j2 b/cns-libs/templates/jenkins/groovy/jenkins_set_no_of_executor.groovy.j2
new file mode 100644
index 00000000..c2b36eb7
--- /dev/null
+++ b/cns-libs/templates/jenkins/groovy/jenkins_set_no_of_executor.groovy.j2
@@ -0,0 +1,5 @@
+import jenkins.model.*
+def instance = Jenkins.getInstance()
+
+// No jobs on master
+instance.setNumExecutors(10)
diff --git a/cns-libs/templates/jenkins/groovy/jenkins_setup_jdk_user.groovy.j2 b/cns-libs/templates/jenkins/groovy/jenkins_setup_jdk_user.groovy.j2
new file mode 100644
index 00000000..905eb1cb
--- /dev/null
+++ b/cns-libs/templates/jenkins/groovy/jenkins_setup_jdk_user.groovy.j2
@@ -0,0 +1,6 @@
+import jenkins.model.*
+import hudson.model.*
+
+def inst = Jenkins.getInstance()
+def desc = inst.getDescriptor("hudson.tools.JDKInstaller")
+println desc.doPostCredential('vsmahajan007@gmail.com','Test#123')
diff --git a/cns-libs/templates/jenkins/jjb/defaults.yaml b/cns-libs/templates/jenkins/jjb/defaults.yaml
new file mode 100644
index 00000000..6434856b
--- /dev/null
+++ b/cns-libs/templates/jenkins/jjb/defaults.yaml
@@ -0,0 +1,2 @@
+- defaults:
+    name: global
diff --git a/cns-libs/templates/jenkins/jjb/job-template.yaml b/cns-libs/templates/jenkins/jjb/job-template.yaml
new file mode 100644
index 00000000..1acab55b
--- /dev/null
+++ b/cns-libs/templates/jenkins/jjb/job-template.yaml
@@ -0,0 +1,9 @@
+- job-template:
+    name: '{name}_job'
+    description: 'job description'
+    project-type: multibranch
+    number-to-keep: 30
+    days-to-keep: 30
+    scm:
+      - git:
+          url: '{git_url}'
diff --git a/cns-libs/templates/jenkins/templates/jenkins-persistent-template.yaml b/cns-libs/templates/jenkins/templates/jenkins-persistent-template.yaml
new file mode 100644
index 00000000..78c89e7d
--- /dev/null
+++ b/cns-libs/templates/jenkins/templates/jenkins-persistent-template.yaml
@@ -0,0 +1,218 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+labels:
+  app: jenkins-persistent
+  template: jenkins-persistent-template
+message: A Jenkins service has been created in your project.  Log into Jenkins with
+  your OpenShift account.  The tutorial at https://github.com/openshift/origin/blob/master/examples/jenkins/README.md
+  contains more information about using this template.
+metadata:
+  annotations:
+    description: |-
+      Jenkins service, with persistent storage.
+
+      NOTE: You must have persistent volumes available in your cluster to use this template.
+    iconClass: icon-jenkins
+    openshift.io/display-name: Jenkins
+    openshift.io/documentation-url: https://docs.openshift.org/latest/using_images/other_images/jenkins.html
+    openshift.io/long-description: This template deploys a Jenkins server capable
+      of managing OpenShift Pipeline builds and supporting OpenShift-based oauth login.
+    openshift.io/provider-display-name: Red Hat, Inc.
+    openshift.io/support-url: https://access.redhat.com
+    tags: instant-app,jenkins
+  creationTimestamp: 2018-03-29T11:18:43Z
+  name: jenkins-persistent
+  namespace: openshift
+  resourceVersion: "898"
+  selfLink: /apis/template.openshift.io/v1/namespaces/openshift/templates/jenkins-persistent
+  uid: f0e942eb-3342-11e8-95ee-0202afd7e94a
+objects:
+- apiVersion: v1
+  kind: Route
+  metadata:
+    annotations:
+      haproxy.router.openshift.io/timeout: 4m
+      template.openshift.io/expose-uri: http://{.spec.host}{.spec.path}
+    name: ${JENKINS_SERVICE_NAME}
+  spec:
+    tls:
+      insecureEdgeTerminationPolicy: Redirect
+      termination: edge
+    to:
+      kind: Service
+      name: ${JENKINS_SERVICE_NAME}
+- apiVersion: v1
+  kind: PersistentVolumeClaim
+  metadata:
+    name: ${JENKINS_SERVICE_NAME}
+  spec:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: ${VOLUME_CAPACITY}
+    storageClassName: ${STORAGE_CLASS_NAME}
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    annotations:
+      template.alpha.openshift.io/wait-for-ready: "true"
+    name: ${JENKINS_SERVICE_NAME}
+  spec:
+    replicas: 1
+    selector:
+      name: ${JENKINS_SERVICE_NAME}
+    strategy:
+      type: Recreate
+    template:
+      metadata:
+        labels:
+          name: ${JENKINS_SERVICE_NAME}
+      spec:
+        containers:
+        - capabilities: {}
+          env:
+          - name: OPENSHIFT_ENABLE_OAUTH
+            value: ${ENABLE_OAUTH}
+          - name: OPENSHIFT_ENABLE_REDIRECT_PROMPT
+            value: "true"
+          - name: KUBERNETES_MASTER
+            value: https://kubernetes.default:443
+          - name: KUBERNETES_TRUST_CERTIFICATES
+            value: "true"
+          - name: JENKINS_SERVICE_NAME
+            value: ${JENKINS_SERVICE_NAME}
+          - name: JNLP_SERVICE_NAME
+            value: ${JNLP_SERVICE_NAME}
+          image: ' '
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 2
+            httpGet:
+              path: /login
+              port: 8080
+            initialDelaySeconds: 420
+            periodSeconds: 360
+            timeoutSeconds: 240
+          name: jenkins
+          readinessProbe:
+            httpGet:
+              path: /login
+              port: 8080
+            initialDelaySeconds: 3
+            timeoutSeconds: 240
+          resources:
+            limits:
+              memory: ${MEMORY_LIMIT}
+          securityContext:
+            capabilities: {}
+            privileged: false
+          terminationMessagePath: /dev/termination-log
+          volumeMounts:
+          - mountPath: /var/lib/jenkins
+            name: ${JENKINS_SERVICE_NAME}-data
+        dnsPolicy: ClusterFirst
+        restartPolicy: Always
+        serviceAccountName: ${JENKINS_SERVICE_NAME}
+        volumes:
+        - name: ${JENKINS_SERVICE_NAME}-data
+          persistentVolumeClaim:
+            claimName: ${JENKINS_SERVICE_NAME}
+    triggers:
+    - imageChangeParams:
+        automatic: true
+        containerNames:
+        - jenkins
+        from:
+          kind: ImageStreamTag
+          name: ${JENKINS_IMAGE_STREAM_TAG}
+          namespace: ${NAMESPACE}
+        lastTriggeredImage: ""
+      type: ImageChange
+    - type: ConfigChange
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    annotations:
+      serviceaccounts.openshift.io/oauth-redirectreference.jenkins: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"${JENKINS_SERVICE_NAME}"}}'
+    name: ${JENKINS_SERVICE_NAME}
+- apiVersion: v1
+  groupNames: null
+  kind: RoleBinding
+  metadata:
+    name: ${JENKINS_SERVICE_NAME}_edit
+  roleRef:
+    name: edit
+  subjects:
+  - kind: ServiceAccount
+    name: ${JENKINS_SERVICE_NAME}
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: ${JNLP_SERVICE_NAME}
+  spec:
+    ports:
+    - name: agent
+      nodePort: 0
+      port: 50000
+      protocol: TCP
+      targetPort: 50000
+    selector:
+      name: ${JENKINS_SERVICE_NAME}
+    sessionAffinity: None
+    type: ClusterIP
+- apiVersion: v1
+  kind: Service
+  metadata:
+    annotations:
+      service.alpha.openshift.io/dependencies: '[{"name": "${JNLP_SERVICE_NAME}",
+        "namespace": "", "kind": "Service"}]'
+      service.openshift.io/infrastructure: "true"
+    name: ${JENKINS_SERVICE_NAME}
+  spec:
+    ports:
+    - name: web
+      nodePort: 0
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+    selector:
+      name: ${JENKINS_SERVICE_NAME}
+    sessionAffinity: None
+    type: ClusterIP
+parameters:
+- description: The name of the OpenShift Service exposed for the Jenkins container.
+  displayName: Jenkins Service Name
+  name: JENKINS_SERVICE_NAME
+  value: jenkins
+- description: The name of the service used for master/slave communication.
+  displayName: Jenkins JNLP Service Name
+  name: JNLP_SERVICE_NAME
+  value: jenkins-jnlp
+- description: Whether to enable OAuth OpenShift integration. If false, the static
+    account 'admin' will be initialized with the password 'password'.
+  displayName: Enable OAuth in Jenkins
+  name: ENABLE_OAUTH
+  value: "true"
+- description: Maximum amount of memory the container can use.
+  displayName: Memory Limit
+  name: MEMORY_LIMIT
+  value: 512Mi
+- description: Volume space available for data, e.g. 512Mi, 2Gi.
+  displayName: Volume Capacity
+  name: VOLUME_CAPACITY
+  required: true
+  value: 1Gi
+- description: The OpenShift Namespace where the Jenkins ImageStream resides.
+  displayName: Jenkins ImageStream Namespace
+  name: NAMESPACE
+  value: openshift
+- description: Name of the ImageStreamTag to be used for the Jenkins image.
+  displayName: Jenkins ImageStreamTag
+  name: JENKINS_IMAGE_STREAM_TAG
+  value: jenkins:2
+- description: Storage Class Name of PVC.
+  displayName: Storage Class Name
+  name: STORAGE_CLASS_NAME
+  required: true
+  value: "gp2"
diff --git a/cns-libs/templates/jenkins/templates/jjb-template.yaml b/cns-libs/templates/jenkins/templates/jjb-template.yaml
new file mode 100644
index 00000000..c26019b1
--- /dev/null
+++ b/cns-libs/templates/jenkins/templates/jjb-template.yaml
@@ -0,0 +1,70 @@
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: config_map_jjb
+objects:
+- apiVersion: v1
+  data:
+    jenkins_jobs.ini: |
+      [jenkins]
+      user=${JENKINS_USER}
+      password=${JENKINS_PASSWORD}
+      url=${JENKINS_URL}
+      query_plugins_info=False
+  kind: ConfigMap
+  metadata:
+    creationTimestamp: null
+    name: ${JJB_SERVICE_NAME}
+- kind: "DeploymentConfig"
+  apiVersion: "v1"
+  metadata:
+    name: ${JJB_SERVICE_NAME}
+  spec:
+    template:
+      metadata:
+        labels:
+          name: ${JJB_SERVICE_NAME}
+      spec:
+        containers:
+          - name: ${JJB_SERVICE_NAME}
+            image: "docker.io/hongkailiu/jjb:2.0.6"
+            env:
+              - name: PYTHONHTTPSVERIFY
+                value: "0"
+              - name: XDG_CACHE_HOME
+                value: "/data"
+            volumeMounts:
+              - name: config-volume
+                mountPath: /etc/jenkins_jobs/
+              - name: "data-vol"
+                mountPath: "/data"
+        volumes:
+          - name: config-volume
+            configMap:
+              name: ${JJB_SERVICE_NAME}
+          - name: "data-vol"
+            emptyDir: {}
+    triggers:
+      - type: "ConfigChange"
+    replicas: 1
+parameters:
+- description: Jenkins User
+  displayName: Jenkins User
+  name: JENKINS_USER
+  required: true
+  value: admin
+- description: Jenkins Password
+  displayName: Jenkins Password
+  name: JENKINS_PASSWORD
+  required: true
+  value: password
+- description: Jenkins URL, eg, https://jenkins-ttt.apps.0327-nbn.qe.rhcloud.com
+  displayName: Jenkins URL
+  name: JENKINS_URL
+  required: true
+- description: JJB Service Name
+  displayName: JJB Service Name
+  name: JJB_SERVICE_NAME
+  required: true
+  value: "jjb"
diff --git a/tests/cns_tests_sample_config.yml b/tests/cns_tests_sample_config.yml
index 52612a96..ce22ac59 100644
--- a/tests/cns_tests_sample_config.yml
+++ b/tests/cns_tests_sample_config.yml
@@ -128,3 +128,17 @@ cns:
         app_pvc_count_dict:
            nginx: 2
 
+scale:
+    - type: jenkins
+      instances: 1
+      namespace: "<fake-namespace-name>"
+      pod_parameters:
+        ENABLE_OAUTH: false
+        MEMORY_LIMIT: "<fake-memory-limit-size>Mi"
+        VOLUME_CAPACITY: "<fake-size-of-gluster-storage>Gi"
+        STORAGE_CLASS_NAME: "fake-name-of-storage-class"
+        JENKINS_IMAGE_STREAM_TAG: "jenkins:2"
+      load_parameters:
+        JOBS: 30
+        ITERATIONS: 30
+        EXECUTORS: 5