Skip to content

N
ngraph
Commit f0c17477 authored by mchrusci's avatar mchrusci Committed by Scott Cyphers
Browse files
Options

[ONNX CI] ONNX CI Improvements (#2026) 

* Kill previous builds on PR update

* Remove Jenkinsfile.groovy

* Lower case method names

* Fix method notify()

* Added comment

Previous build deletion workaround to be removed as soon as better, less security vulnerable solution is found.

* Fix inconsistent stage names
parent b04f3c36
Hide whitespace changes
Inline Side-by-side
Showing with 54 additions and 226 deletions
.ci/onnx/jenkins/Jenkinsfile
View file @ f0c17477
......@@ -30,7 +30,27 @@ GIT_COMMIT_AUTHOR_EMAIL=""
GIT_COMMIT_HASH=""
GIT_COMMIT_SUBJECT=""
def CloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
// workaround for aborting previous builds on PR update
// TODO: Move to plugin based solution as soon as it's available
@NonCPS
def killPreviousRunningJobs() {
def jobname = env.JOB_NAME
def buildnum = env.BUILD_NUMBER.toInteger()
def job = Jenkins.instance.getItemByFullName(jobname)
for (build in job.builds) {
if (!build.isBuilding()){
continue;
}
if (buildnum == build.getNumber().toInteger()){
continue;
}
echo "Kill task = ${build}"
build.doStop();
}
}
def cloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
stage('Clone Repos') {
try {
sh "git clone $onnx_git_address -b $CHANGE_BRANCH ."
......@@ -61,7 +81,7 @@ def CloneRepository(String jenkins_github_credential_id, String ngraph_git_addre
}
}
def BuildImage(configurationMaps) {
def buildImage(configurationMaps) {
Closure buildMethod = { configMap ->
sh """
${CI_ROOT}/utils/docker.sh build \
......@@ -70,12 +90,12 @@ def BuildImage(configurationMaps) {
--dockerfile_path=${configMap["dockerfilePath"]}
"""
}
UTILS.CreateStage("Build_Image", buildMethod, configurationMaps)
UTILS.createStage("Build_image", buildMethod, configurationMaps)
}
def RunDockerContainers(configurationMaps) {
def runDockerContainers(configurationMaps) {
Closure runContainerMethod = { configMap ->
UTILS.PropagateStatus("Build_Image", configMap["name"])
UTILS.propagateStatus("Build_image", configMap["name"])
sh """
mkdir -p ${HOME}/ONNX_CI
${CI_ROOT}/utils/docker.sh start \
......@@ -85,32 +105,32 @@ def RunDockerContainers(configurationMaps) {
--volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root"
"""
}
UTILS.CreateStage("Run_docker_containers", runContainerMethod, configurationMaps)
UTILS.createStage("Run_docker_containers", runContainerMethod, configurationMaps)
}
def PrepareEnvironment(configurationMaps) {
def prepareEnvironment(configurationMaps) {
Closure prepareEnvironmentMethod = { configMap ->
UTILS.PropagateStatus("Run_docker_containers", configMap["dockerContainerName"])
UTILS.propagateStatus("Run_docker_containers", configMap["dockerContainerName"])
sh """
docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home
docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh"
"""
}
UTILS.CreateStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
UTILS.createStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
}
def RunToxTests(configurationMaps) {
def runToxTests(configurationMaps) {
Closure runToxTestsMethod = { configMap ->
UTILS.PropagateStatus("Prepare_environment", configMap["dockerContainerName"])
UTILS.propagateStatus("Prepare_environment", configMap["dockerContainerName"])
sh """
NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl')
docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/
"""
}
UTILS.CreateStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
UTILS.createStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
}
def Cleanup(configurationMaps) {
def cleanup(configurationMaps) {
Closure cleanupMethod = { configMap ->
sh """
cd ${HOME}/ONNX_CI
......@@ -122,13 +142,13 @@ def Cleanup(configurationMaps) {
rm -rf ${WORKSPACE}/${BUILD_NUMBER}
"""
}
UTILS.CreateStage("Cleanup", cleanupMethod, configurationMaps)
UTILS.createStage("Cleanup", cleanupMethod, configurationMaps)
}
def Notify() {
def notifyByEmail() {
configurationMaps = []
configurationMaps.add([
"name": "notify"
"name": "Notify"
])
String notifyPeople = "$GIT_PR_AUTHOR_EMAIL, $GIT_COMMIT_AUTHOR_EMAIL"
Closure notifyMethod = { configMap ->
......@@ -151,27 +171,28 @@ def Notify() {
)
}
}
UTILS.CreateStage("Notify", notifyMethod, configurationMaps)
UTILS.createStage("Notify", notifyMethod, configurationMaps)
}
def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
killPreviousRunningJobs()
node(label) {
timeout(activity: true, time: 15) {
WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
def configurationMaps;
try {
dir ("${WORKDIR}") {
CloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
cloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
// Load CI API
UTILS = load "${CI_ROOT}/utils/utils.groovy"
result = 'SUCCESS'
// Create configuration maps
configurationMaps = UTILS.GetDockerEnvList(projectName, dockerContainerName, projectRoot)
configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
// Execute CI steps
BuildImage(configurationMaps)
RunDockerContainers(configurationMaps)
PrepareEnvironment(configurationMaps)
RunToxTests(configurationMaps)
buildImage(configurationMaps)
runDockerContainers(configurationMaps)
prepareEnvironment(configurationMaps)
runToxTests(configurationMaps)
}
}
catch(e) {
......@@ -183,8 +204,8 @@ def main(String label, String projectName, String projectRoot, String dockerCont
}
}
finally {
Cleanup(configurationMaps)
Notify()
cleanup(configurationMaps)
notifyByEmail()
}
}
}
......
.ci/onnx/jenkins/Jenkinsfile.groovy deleted 100644 → 0
View file @ b04f3c36
// INTEL CONFIDENTIAL
// Copyright 2018 Intel Corporation All Rights Reserved.
// The source code contained or described herein and all documents related to the
// source code ("Material") are owned by Intel Corporation or its suppliers or
// licensors. Title to the Material remains with Intel Corporation or its
// suppliers and licensors. The Material may contain trade secrets and proprietary
// and confidential information of Intel Corporation and its suppliers and
// licensors, and is protected by worldwide copyright and trade secret laws and
// treaty provisions. No part of the Material may be used, copied, reproduced,
// modified, published, uploaded, posted, transmitted, distributed, or disclosed
// in any way without Intel's prior express written permission.
// No license under any patent, copyright, trade secret or other intellectual
// property right is granted to or conferred upon you by disclosure or delivery of
// the Materials, either expressly, by implication, inducement, estoppel or
// otherwise. Any license under such intellectual property rights must be express
// and approved by Intel in writing.
// Set LABEL variable if empty or not declared
try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
// CI settings and constants
PROJECT_NAME = "ngraph-onnx"
CI_ROOT = "ngraph/.ci/onnx/jenkins"
DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci"
NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
ONNX_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph-onnx.git"
JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
ONNX_BRANCH = "$CHANGE_BRANCH"
GIT_PR_AUTHOR_EMAIL=""
GIT_COMMIT_AUTHOR_EMAIL=""
GIT_COMMIT_HASH=""
GIT_COMMIT_SUBJECT=""
def CloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
stage('Clone Repos') {
try {
sh "git clone $onnx_git_address -b $CHANGE_BRANCH ."
}
catch (Exception e) {
ONNX_BRANCH = "master"
sh """
echo "WARNING! Failed to clone ngraph-onnx branch $CHANGE_BRANCH ! Falling back to master."
echo "EXCEPTION: $e"
"""
checkout([$class: 'GitSCM',
branches: [[name: "master"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
url: "${onnx_git_address}"]]])
}
dir ("ngraph") {
checkout([$class: 'GitSCM',
branches: [[name: "$CHANGE_BRANCH"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
url: "${ngraph_git_address}"]]])
GIT_PR_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
GIT_COMMIT_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
GIT_COMMIT_HASH = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
GIT_COMMIT_SUBJECT = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
}
}
}
def BuildImage(configurationMaps) {
Closure buildMethod = { configMap ->
sh """
${CI_ROOT}/utils/docker.sh build \
--name=${configMap["projectName"]} \
--version=${configMap["name"]} \
--dockerfile_path=${configMap["dockerfilePath"]}
"""
}
UTILS.CreateStage("Build_Image", buildMethod, configurationMaps)
}
def RunDockerContainers(configurationMaps) {
Closure runContainerMethod = { configMap ->
UTILS.PropagateStatus("Build_Image", configMap["name"])
sh """
mkdir -p ${HOME}/ONNX_CI
${CI_ROOT}/utils/docker.sh start \
--name=${configMap["projectName"]} \
--version=${configMap["name"]} \
--container_name=${configMap["dockerContainerName"]} \
--volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root"
"""
}
UTILS.CreateStage("Run_docker_containers", runContainerMethod, configurationMaps)
}
def PrepareEnvironment(configurationMaps) {
Closure prepareEnvironmentMethod = { configMap ->
UTILS.PropagateStatus("Run_docker_containers", configMap["dockerContainerName"])
sh """
docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home
docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh"
"""
}
UTILS.CreateStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
}
def RunToxTests(configurationMaps) {
Closure runToxTestsMethod = { configMap ->
UTILS.PropagateStatus("Prepare_environment", configMap["dockerContainerName"])
sh """
NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl')
docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/
"""
}
UTILS.CreateStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
}
def Cleanup(configurationMaps) {
Closure cleanupMethod = { configMap ->
sh """
cd ${HOME}/ONNX_CI
./docker.sh chmod --container_name=${configMap["dockerContainerName"]} --directory="/logs" --options="-R 777" || true
./docker.sh stop --container_name=${configMap["dockerContainerName"]} || true
./docker.sh remove --container_name=${configMap["dockerContainerName"]} || true
./docker.sh clean_up || true
rm ${HOME}/ONNX_CI/docker.sh
rm -rf ${WORKSPACE}/${BUILD_NUMBER}
"""
}
UTILS.CreateStage("Cleanup", cleanupMethod, configurationMaps)
}
def Notify() {
configurationMaps = []
configurationMaps.add([
"name": "notify"
])
String notifyPeople = "$GIT_PR_AUTHOR_EMAIL, $GIT_COMMIT_AUTHOR_EMAIL"
Closure notifyMethod = { configMap ->
if(currentBuild.result == "FAILURE") {
emailext (
subject: "NGraph-Onnx CI: NGraph PR $CHANGE_ID $currentBuild.result!",
body: """
<table style="width:100%">
<tr><td>Status:</td> <td>${currentBuild.result}</td></tr>
<tr><td>Pull Request Title:</td> <td>$CHANGE_TITLE</td></tr>
<tr><td>Pull Request:</td> <td><a href=$CHANGE_URL>$CHANGE_ID</a> </td></tr>
<tr><td>Branch:</td> <td>$CHANGE_BRANCH</td></tr>
<tr><td>Commit Hash:</td> <td>$GIT_COMMIT_SUBJECT</td></tr>
<tr><td>Commit Subject:</td> <td>$GIT_COMMIT_HASH</td></tr>
<tr><td>Jenkins Build:</td> <td> <a href=$RUN_DISPLAY_URL> ${BUILD_NUMBER} </a> </td></tr>
<tr><td>nGraph-ONNX Branch:</td> <td>${ONNX_BRANCH}</td></tr>
</table>
""",
to: "${notifyPeople}"
)
}
}
UTILS.CreateStage("Notify", notifyMethod, configurationMaps)
}
def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
node(label) {
timeout(activity: true, time: 15) {
WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
def configurationMaps;
try {
dir ("${WORKDIR}") {
CloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
// Load CI API
UTILS = load "${CI_ROOT}/utils/utils.groovy"
result = 'SUCCESS'
// Create configuration maps
configurationMaps = UTILS.GetDockerEnvList(projectName, dockerContainerName, projectRoot)
// Execute CI steps
BuildImage(configurationMaps)
RunDockerContainers(configurationMaps)
PrepareEnvironment(configurationMaps)
RunToxTests(configurationMaps)
}
}
catch(e) {
// Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
if ("$e".contains("143")) {
currentBuild.result = "ABORTED"
} else {
currentBuild.result = "FAILURE"
}
}
finally {
Cleanup(configurationMaps)
Notify()
}
}
}
}
main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS, ONNX_GIT_ADDRESS)
.ci/onnx/jenkins/utils/utils.groovy
View file @ f0c17477
......@@ -16,7 +16,7 @@
// and approved by Intel in writing.
STAGES_STATUS_MAP = [:]
def GetDockerEnvList(String projectName, String dockerContainerNamePrefix, String projectRoot = projectName) {
def getDockerEnvList(String projectName, String dockerContainerNamePrefix, String projectRoot = projectName) {
/**
* This method generates configuration map list using dockerfiles available in dockerfiles directory
*
......@@ -38,7 +38,7 @@ def GetDockerEnvList(String projectName, String dockerContainerNamePrefix, Strin
return envList
}
def GenerateMap(Closure method, configurationMaps) {
def generateMap(Closure method, configurationMaps) {
/**
* Generates map for method using configurationMaps.
*
......@@ -56,7 +56,7 @@ def GenerateMap(Closure method, configurationMaps) {
return executionMap
}
def CreateStage(String stageName, Closure method, configurationMaps) {
def createStage(String stageName, Closure method, configurationMaps) {
/**
* Create pipeline stage.
*
......@@ -79,12 +79,12 @@ def CreateStage(String stageName, Closure method, configurationMaps) {
status = "FAILURE"
throw e
} finally {
UTILS.SetConfigurationStatus(configMap["stageName"], configMap["name"], status)
UTILS.setConfigurationStatus(configMap["stageName"], configMap["name"], status)
}
}
try {
def prepareEnvMap = GenerateMap(genericBodyMethod, configurationMaps)
def prepareEnvMap = generateMap(genericBodyMethod, configurationMaps)
parallel prepareEnvMap
} catch(e) {
// Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
......@@ -97,7 +97,7 @@ def CreateStage(String stageName, Closure method, configurationMaps) {
}
}
def SetConfigurationStatus(String stageName, String configurationName, String status) {
def setConfigurationStatus(String stageName, String configurationName, String status) {
/**
* Set stage status.
*
......@@ -115,7 +115,7 @@ def SetConfigurationStatus(String stageName, String configurationName, String st
}
}
def PropagateStatus(String parentStageName, String parentConfigurationName) {
def propagateStatus(String parentStageName, String parentConfigurationName) {
/**
* Popagate status in parent configuration fails.
* This method will throw exeption "Propagating status of $parentStageName"
......@@ -131,7 +131,7 @@ def PropagateStatus(String parentStageName, String parentConfigurationName) {
}
}
def ShowStatusMap() {
def showStatusMap() {
/**
* Display status map for every defined stage.
*/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment