Jenkinsfile

// INTEL CONFIDENTIAL
// Copyright 2018-2019 Intel Corporation All Rights Reserved.
// The source code contained or described herein and all documents related to the
// source code ("Material") are owned by Intel Corporation or its suppliers or
// licensors. Title to the Material remains with Intel Corporation or its
// suppliers and licensors. The Material may contain trade secrets and proprietary
// and confidential information of Intel Corporation and its suppliers and
// licensors, and is protected by worldwide copyright and trade secret laws and
// treaty provisions. No part of the Material may be used, copied, reproduced,
// modified, published, uploaded, posted, transmitted, distributed, or disclosed
// in any way without Intel's prior express written permission.
// No license under any patent, copyright, trade secret or other intellectual
// property right is granted to or conferred upon you by disclosure or delivery of
// the Materials, either expressly, by implication, inducement, estoppel or
// otherwise. Any license under such intellectual property rights must be express
// and approved by Intel in writing.

// Set LABEL variable if empty or not declared
try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
// CI settings and constants
PROJECT_NAME = "ngraph-onnx"
CI_ROOT = "ngraph/.ci/onnx/jenkins"
DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci"
NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
ONNX_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph-onnx.git"
JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
ONNX_BRANCH = "$CHANGE_BRANCH"
GIT_PR_AUTHOR_EMAIL=""
GIT_COMMIT_AUTHOR_EMAIL=""
GIT_COMMIT_HASH=""
GIT_COMMIT_SUBJECT=""

// workaround for aborting previous builds on PR update
// TODO: Move to plugin based solution as soon as it's available
@NonCPS
def killPreviousRunningJobs() {
    def jobname = env.JOB_NAME
    def buildnum = env.BUILD_NUMBER.toInteger()

    def job = Jenkins.instance.getItemByFullName(jobname)
    for (build in job.builds) {
        if (!build.isBuilding()){
            continue;
        }
        if (buildnum == build.getNumber().toInteger()){
            continue;
        }
        echo "Kill task = ${build}"
        build.doStop();
    }
}

def cloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
    stage('Clone Repos') {
        try {
            sh "git clone $onnx_git_address -b $CHANGE_BRANCH ."
        }
        catch (Exception e) {
            ONNX_BRANCH = "master"
            sh """
                echo "WARNING! Failed to clone ngraph-onnx branch $CHANGE_BRANCH ! Falling back to master."
                echo "EXCEPTION: $e"
            """
            checkout([$class: 'GitSCM',
                branches: [[name: "master"]],
                doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
                userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
                url: "${onnx_git_address}"]]])
        }
        dir ("ngraph") {
                checkout([$class: 'GitSCM',
                    branches: [[name: "$CHANGE_BRANCH"]],
                    doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
                    userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
                    url: "${ngraph_git_address}"]]])
                GIT_PR_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
                GIT_COMMIT_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
                GIT_COMMIT_HASH = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
                GIT_COMMIT_SUBJECT = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
        }
    }
}

def buildImage(configurationMaps) {
    Closure buildMethod = { configMap ->
        sh """
            ${CI_ROOT}/utils/docker.sh build \
                                --name=${configMap["projectName"]} \
                                --version=${configMap["name"]} \
                                --dockerfile_path=${configMap["dockerfilePath"]}
        """
    }
    UTILS.createStage("Build_image", buildMethod, configurationMaps)
}

def runDockerContainers(configurationMaps) {
    Closure runContainerMethod = { configMap ->
        UTILS.propagateStatus("Build_image", configMap["name"])
        sh """
            mkdir -p ${HOME}/ONNX_CI
            ${CI_ROOT}/utils/docker.sh start \
                                --name=${configMap["projectName"]} \
                                --version=${configMap["name"]} \
                                --container_name=${configMap["dockerContainerName"]} \
                                --volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root"
        """
    }
    UTILS.createStage("Run_docker_containers", runContainerMethod, configurationMaps)
}

def prepareEnvironment(configurationMaps) {
    Closure prepareEnvironmentMethod = { configMap ->
        UTILS.propagateStatus("Run_docker_containers", configMap["dockerContainerName"])
        sh """
            docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home
            docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh"
        """
    }
    UTILS.createStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
}

def runToxTests(configurationMaps) {
    Closure runToxTestsMethod = { configMap ->
        UTILS.propagateStatus("Prepare_environment", configMap["dockerContainerName"])
        sh """
            NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl')
            docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/
        """
    }
    UTILS.createStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
}

def cleanup(configurationMaps) {
    Closure cleanupMethod = { configMap ->
        sh """
            cd ${HOME}/ONNX_CI
            ./docker.sh chmod --container_name=${configMap["dockerContainerName"]} --directory="/logs" --options="-R 777" || true
            ./docker.sh stop --container_name=${configMap["dockerContainerName"]} || true
            ./docker.sh remove --container_name=${configMap["dockerContainerName"]} || true
            ./docker.sh clean_up || true
            rm ${HOME}/ONNX_CI/docker.sh
            rm -rf ${WORKSPACE}/${BUILD_NUMBER}
        """
    }
    UTILS.createStage("Cleanup", cleanupMethod, configurationMaps)
}

def notifyByEmail() {
    configurationMaps = []
    configurationMaps.add([
        "name": "Notify"
    ])
    String notifyPeople = "$GIT_PR_AUTHOR_EMAIL, $GIT_COMMIT_AUTHOR_EMAIL"
    Closure notifyMethod = { configMap ->
        if(currentBuild.result == "FAILURE") {
            emailext (
                subject: "NGraph-Onnx CI: NGraph PR $CHANGE_ID $currentBuild.result!",
                body: """
                    <table style="width:100%">
                        <tr><td>Status:</td> <td>${currentBuild.result}</td></tr>
                        <tr><td>Pull Request Title:</td> <td>$CHANGE_TITLE</td></tr>
                        <tr><td>Pull Request:</td> <td><a href=$CHANGE_URL>$CHANGE_ID</a> </td></tr>
                        <tr><td>Branch:</td> <td>$CHANGE_BRANCH</td></tr>
                        <tr><td>Commit Hash:</td> <td>$GIT_COMMIT_SUBJECT</td></tr>
                        <tr><td>Commit Subject:</td> <td>$GIT_COMMIT_HASH</td></tr>
                        <tr><td>Jenkins Build:</td> <td> <a href=$RUN_DISPLAY_URL> ${BUILD_NUMBER} </a> </td></tr>
                        <tr><td>nGraph-ONNX Branch:</td> <td>${ONNX_BRANCH}</td></tr>
                    </table>
                """,
                to: "${notifyPeople}"
            )
        }
    }
    UTILS.createStage("Notify", notifyMethod, configurationMaps)
}

def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
    killPreviousRunningJobs()
    node(label) {
        timeout(activity: true, time: 15) {
            WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
            def configurationMaps;
            try {
                dir ("${WORKDIR}") {
                    cloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
                    // Load CI API
                    UTILS = load "${CI_ROOT}/utils/utils.groovy"
                    result = 'SUCCESS'
                    // Create configuration maps
                    configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
                    // Execute CI steps
                    buildImage(configurationMaps)
                    runDockerContainers(configurationMaps)
                    prepareEnvironment(configurationMaps)
                    runToxTests(configurationMaps)
                }
            }
            catch(e) {
                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
                if ("$e".contains("143")) {
                    currentBuild.result = "ABORTED"
                } else {
                    currentBuild.result = "FAILURE"
                }
            }
            finally {
                cleanup(configurationMaps)
                notifyByEmail()
            }
        }
    }
}

main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS, ONNX_GIT_ADDRESS)