• mchrusci's avatar
    [ONNX CI] nGraph master as docker image (#2676) · 5473a18a
    mchrusci authored
    * Build branch from scratch if building upon cached master fails
    
    * Revert "Build branch from scratch if building upon cached master fails"
    
    This reverts commit 0141d4151d0fbfd76733708833180e36adbd51b2.
    
    * Revert "Revert "Build branch from scratch if building upon cached master fails""
    
    This reverts commit 64cf85c101664544a1db8736bc6ae35cb075ef77.
    
    * Update ONNX CI workflow
    
    - Periodically build base nGraph image in a separate job
    - Use that base to build branches in CI
    - If that fails - build from scratch
    
    * Moved base builder script
    
    * Fixed ngraph repo path
    
    * Fix branch name
    
    * Fix CI root path
    
    * Docker login must be done explicitly
    
    * Fix credentials variable
    
    * Fix pullImage method name
    
    * Removed Docker registry addresses
    
    * Set default branch to master
    
    * Clone branch specified as parameter
    
    * Fix try catch error
    
    * Fix build script execution
    
    * Added docker_registry parameter to start container script
    
    * Calling setConfigurationStatus directly
    
    * Revert "Calling setConfigurationStatus directly"
    
    This reverts commit ea16a54ed1919f08bf0e5e1c4e8284146bcbaf6d.
    
    * Fix missing $
    
    * Fix password redirection to stdin
    
    * Removed internal address
    
    * Fix closing bracket
    5473a18a
Jenkinsfile 4.11 KB
// Set LABEL variable if empty or not declared
try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
try{ if(BRANCH.trim() == "") {throw new Exception();} }catch(Exception e){BRANCH="master"}; echo "${BRANCH}"
if(DOCKER_REGISTRY.trim() == "") {throw new Exception("No Docker registry specified!");}
// CI settings and constants
PROJECT_NAME = "ngraph_cpp"
CI_ROOT = ".ci/onnx/jenkins"
DOCKER_CONTAINER_NAME = "jenkins_ngraph-onnx_ci"
NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"

def cloneRepository(String jenkins_github_credential_id, String ngraph_git_address) {
    stage('Clone Repo') {
        checkout([$class: 'GitSCM',
            branches: [[name: "${BRANCH}"]],
            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
            userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
            url: "${ngraph_git_address}"]]])
    }
}

def buildImage(configurationMaps) {
    Closure buildMethod = { configMap ->
        sh """
            ${CI_ROOT}/utils/docker.sh build \
                                --docker_registry=${DOCKER_REGISTRY} \
                                --name=${configMap["projectName"]} \
                                --version=${configMap["name"]} \
                                --dockerfile_path=${configMap["dockerfilePath"]} || return 1
        """
    }
    UTILS.createStage("Build_image", buildMethod, configurationMaps)
}

def pushImage(configurationMaps) {
    Closure pushMethod = { configMap ->
        UTILS.propagateStatus("Build_image", configMap["name"])
        withCredentials([usernamePassword(credentialsId: "${DOCKER_CREDENTIALS}",
                                          usernameVariable: 'DOCKER_USERNAME',
                                          passwordVariable: 'DOCKER_PASSWORD')]) {
            sh """
                docker login ${DOCKER_REGISTRY} --username ${DOCKER_USERNAME} --password-stdin <<< \${DOCKER_PASSWORD}
                ${CI_ROOT}/utils/docker.sh push \
                                    --docker_registry=${DOCKER_REGISTRY} \
                                    --name=${configMap["projectName"]} \
                                    --version=${configMap["name"]} || return 1
            """
        }
    }
    UTILS.createStage("Push_image", pushMethod, configurationMaps)
}

def cleanup(configurationMaps) {
    Closure cleanupMethod = { configMap ->
        sh """
            rm -rf ${WORKSPACE}/${BUILD_NUMBER}
        """
    }
    UTILS.createStage("Cleanup", cleanupMethod, configurationMaps)
}

def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address) {
    node(label) {
        timeout(activity: true, time: 15) {
            WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
            def configurationMaps;
            try {
                dir ("${WORKDIR}") {
                    cloneRepository(jenkins_github_credential_id, ngraph_git_address)
                    // Load CI API
                    UTILS = load "${CI_ROOT}/utils/utils.groovy"
                    result = 'SUCCESS'
                    // Create configuration maps
                    configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
                    // Build and push base images
                    buildImage(configurationMaps)
                    pushImage(configurationMaps)
                }
            }
            catch(e) {
                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
                if ("$e".contains("143")) {
                    currentBuild.result = "ABORTED"
                } else {
                    currentBuild.result = "FAILURE"
                }
            }
            finally {
                cleanup(configurationMaps)
            }
        }
    }
}

main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS)