// INTEL CONFIDENTIAL // Copyright 2018 Intel Corporation All Rights Reserved. // The source code contained or described herein and all documents related to the // source code ("Material") are owned by Intel Corporation or its suppliers or // licensors. Title to the Material remains with Intel Corporation or its // suppliers and licensors. The Material may contain trade secrets and proprietary // and confidential information of Intel Corporation and its suppliers and // licensors, and is protected by worldwide copyright and trade secret laws and // treaty provisions. No part of the Material may be used, copied, reproduced, // modified, published, uploaded, posted, transmitted, distributed, or disclosed // in any way without Intel's prior express written permission. // No license under any patent, copyright, trade secret or other intellectual // property right is granted to or conferred upon you by disclosure or delivery of // the Materials, either expressly, by implication, inducement, estoppel or // otherwise. Any license under such intellectual property rights must be express // and approved by Intel in writing. // Set LABEL variable if empty or not declared try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}" // CI settings and constants PROJECT_NAME = "ngraph-onnx" CI_ROOT = "ngraph/.ci/onnx/jenkins" DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci" NGRAPH_GIT_ADDRESS = "https://www.github.com/NervanaSystems/ngraph.git" ONNX_GIT_ADDRESS = "https://www.github.com/NervanaSystems/ngraph-onnx.git" JENKINS_GITHUB_CREDENTIAL_ID = "6887a177-8c4d-4fe9-9c3b-fcd71b22bfba" WARNING = "" def CloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) { stage('Clone Repos') { try { checkout([$class: 'GitSCM', branches: [[name: "$ghprbSourceBranch"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}", url: "${onnx_git_address}"]]]) } catch (Exception e) { WARNING = "WARNING! Failed to clone ngraph-onnx branch $ghprbSourceBranch ! Falling back to master." sh """ echo ${WARNING} """ checkout([$class: 'GitSCM', branches: [[name: "master"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}", url: "${onnx_git_address}"]]]) } dir ("ngraph") { checkout([$class: 'GitSCM', branches: [[name: "$ghprbSourceBranch"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}", url: "${ngraph_git_address}"]]]) } } } def BuildImage(configurationMaps) { Closure buildMethod = { configMap -> sh """ ${CI_ROOT}/utils/docker.sh build \ --name=${configMap["projectName"]} \ --version=${configMap["name"]} \ --dockerfile_path=${configMap["dockerfilePath"]} """ } UTILS.CreateStage("Build_Image", buildMethod, configurationMaps) } def RunDockerContainers(configurationMaps) { Closure runContainerMethod = { configMap -> UTILS.PropagateStatus("Build_Image", configMap["name"]) sh """ mkdir -p ${HOME}/ONNX_CI ${CI_ROOT}/utils/docker.sh start \ --name=${configMap["projectName"]} \ --version=${configMap["name"]} \ --container_name=${configMap["dockerContainerName"]} \ --volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root" """ } UTILS.CreateStage("Run_docker_containers", runContainerMethod, configurationMaps) } def PrepareEnvironment(configurationMaps) { Closure prepareEnvironmentMethod = { configMap -> UTILS.PropagateStatus("Run_docker_containers", configMap["dockerContainerName"]) sh """ docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh --ngraph-branch=$ghprbSourceBranch" """ } UTILS.CreateStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps) } def RunToxTests(configurationMaps) { Closure runToxTestsMethod = { configMap -> UTILS.PropagateStatus("Prepare_environment", configMap["dockerContainerName"]) sh """ NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl') docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/ mkdir -p ${HOME}/ONNX_CI/onnx_models/ if [ -e ${WORKDIR}/.onnx ]; then docker cp ${configMap["dockerContainerName"]}:/root/.onnx/ ~/ONNX_CI/onnx_models/ fi """ } UTILS.CreateStage("Run_tox_tests", runToxTestsMethod, configurationMaps) } def Cleanup(configurationMaps) { Closure cleanupMethod = { configMap -> sh """ cd ${HOME}/ONNX_CI ./docker.sh chmod --container_name=${configMap["dockerContainerName"]} --directory="/logs" --options="-R 777" || true docker exec ${configMap["dockerContainerName"]} rm -rf /root/* || true ./docker.sh stop --container_name=${configMap["dockerContainerName"]} || true ./docker.sh remove --container_name=${configMap["dockerContainerName"]} || true ./docker.sh clean_up || true rm ${HOME}/ONNX_CI/docker.sh rm -rf ${WORKSPACE}/${BUILD_NUMBER} """ } UTILS.CreateStage("Cleanup", cleanupMethod, configurationMaps) } def Notify() { configurationMaps = [] configurationMaps.add([ "name": "notify" ]) String notifyPeople = "$ghprbPullAuthorEmail, $ghprbActualCommitAuthorEmail" Closure notifyMethod = { configMap -> if(currentBuild.result != "FAILURE") { currentBuild.result = "SUCCESS" } if(currentBuild.result == "FAILURE" || WARNING != "") { emailext ( subject: "NGraph-Onnx CI: NGraph PR $ghprbPullId $currentBuild.result!", body: """ <table style="width:100%"> <tr><td>${WARNING}</td></tr> <tr><td>Status:</td> <td>${currentBuild.result}</td></tr> <tr><td>Repository</td> <td>$ghprbGhRepository</td></tr> <tr><td>Branch:</td> <td>$ghprbSourceBranch</td></tr> <tr><td>Pull Request:</td> <td>$ghprbPullId</td></tr> <tr><td>Commit SHA:</td> <td>$ghprbActualCommit</td></tr> <tr><td>Link:</td> <td>$ghprbPullLink</td></tr> </table> """, to: "${notifyPeople}" ) } } UTILS.CreateStage("Notify", notifyMethod, configurationMaps) } def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) { node(label) { timeout(activity: true, time: 60) { WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}" def configurationMaps; try { dir ("${WORKDIR}") { CloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address) // Load CI API UTILS = load "${CI_ROOT}/utils/utils.groovy" result = 'SUCCESS' // Create configuration maps configurationMaps = UTILS.GetDockerEnvList(projectName, dockerContainerName, projectRoot) // Execute CI steps BuildImage(configurationMaps) RunDockerContainers(configurationMaps) PrepareEnvironment(configurationMaps) RunToxTests(configurationMaps) } } finally { Cleanup(configurationMaps) Notify() } } } } main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS, ONNX_GIT_ADDRESS)