[ONNX CI] nGraph master as docker image (#2676)

* Build branch from scratch if building upon cached master fails

* Revert "Build branch from scratch if building upon cached master fails"

This reverts commit 0141d4151d0fbfd76733708833180e36adbd51b2.

* Revert "Revert "Build branch from scratch if building upon cached master fails""

This reverts commit 64cf85c101664544a1db8736bc6ae35cb075ef77.

* Update ONNX CI workflow

- Periodically build base nGraph image in a separate job
- Use that base to build branches in CI
- If that fails - build from scratch

* Moved base builder script

* Fixed ngraph repo path

* Fix branch name

* Fix CI root path

* Docker login must be done explicitly

* Fix credentials variable

* Fix pullImage method name

* Removed Docker registry addresses

* Set default branch to master

* Clone branch specified as parameter

* Fix try catch error

* Fix build script execution

* Added docker_registry parameter to start container script

* Calling setConfigurationStatus directly

* Revert "Calling setConfigurationStatus directly"

This reverts commit ea16a54ed1919f08bf0e5e1c4e8284146bcbaf6d.

* Fix missing $

* Fix password redirection to stdin

* Removed internal address

* Fix closing bracket

.ci/onnx/jenkins/Jenkinsfile

@@ -17,10 +17,11 @@
 
 // Set LABEL variable if empty or not declared
 try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
+if(DOCKER_REGISTRY.trim() == "") {throw new Exception("Missing Docker registry url!");}
 // CI settings and constants
-PROJECT_NAME = "ngraph-onnx"
+PROJECT_NAME = "ngraph_cpp"
 CI_ROOT = "ngraph/.ci/onnx/jenkins"
-DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci"
+DOCKER_CONTAINER_NAME = "jenkins_ngraph-onnx_ci"
 NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
 ONNX_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph-onnx.git"
 JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
@@ -81,24 +82,25 @@ def cloneRepository(String jenkins_github_credential_id, String ngraph_git_addre
     }
 }
 
-def buildImage(configurationMaps) {
-    Closure buildMethod = { configMap ->
+def pullImage(configurationMaps) {
+    Closure pullMethod = { configMap ->
         sh """
-            ${CI_ROOT}/utils/docker.sh build \
+            ${CI_ROOT}/utils/docker.sh pull \
+                                --docker_registry=${DOCKER_REGISTRY} \
                                 --name=${configMap["projectName"]} \
-                                --version=${configMap["name"]} \
-                                --dockerfile_path=${configMap["dockerfilePath"]}
+                                --version=${configMap["name"]} || return 1
         """
     }
-    UTILS.createStage("Build_image", buildMethod, configurationMaps)
+    UTILS.createStage("Pull_image", pullMethod, configurationMaps)
 }
 
 def runDockerContainers(configurationMaps) {
     Closure runContainerMethod = { configMap ->
-        UTILS.propagateStatus("Build_image", configMap["name"])
+        UTILS.propagateStatus("Pull_image", configMap["name"])
         sh """
             mkdir -p ${HOME}/ONNX_CI
             ${CI_ROOT}/utils/docker.sh start \
+                                --docker_registry=${DOCKER_REGISTRY} \
                                 --name=${configMap["projectName"]} \
                                 --version=${configMap["name"]} \
                                 --container_name=${configMap["dockerContainerName"]} \
@@ -189,7 +191,7 @@ def main(String label, String projectName, String projectRoot, String dockerCont
                     // Create configuration maps
                     configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
                     // Execute CI steps
-                    buildImage(configurationMaps)
+                    pullImage(configurationMaps)
                     runDockerContainers(configurationMaps)
                     prepareEnvironment(configurationMaps)
                     runToxTests(configurationMaps)

.ci/onnx/jenkins/base_image_builder/Jenkinsfile

+// Set LABEL variable if empty or not declared
+try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
+try{ if(BRANCH.trim() == "") {throw new Exception();} }catch(Exception e){BRANCH="master"}; echo "${BRANCH}"
+if(DOCKER_REGISTRY.trim() == "") {throw new Exception("No Docker registry specified!");}
+// CI settings and constants
+PROJECT_NAME = "ngraph_cpp"
+CI_ROOT = ".ci/onnx/jenkins"
+DOCKER_CONTAINER_NAME = "jenkins_ngraph-onnx_ci"
+NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
+JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
+
+def cloneRepository(String jenkins_github_credential_id, String ngraph_git_address) {
+    stage('Clone Repo') {
+        checkout([$class: 'GitSCM',
+            branches: [[name: "${BRANCH}"]],
+            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
+            userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
+            url: "${ngraph_git_address}"]]])
+    }
+}
+
+def buildImage(configurationMaps) {
+    Closure buildMethod = { configMap ->
+        sh """
+            ${CI_ROOT}/utils/docker.sh build \
+                                --docker_registry=${DOCKER_REGISTRY} \
+                                --name=${configMap["projectName"]} \
+                                --version=${configMap["name"]} \
+                                --dockerfile_path=${configMap["dockerfilePath"]} || return 1
+        """
+    }
+    UTILS.createStage("Build_image", buildMethod, configurationMaps)
+}
+
+def pushImage(configurationMaps) {
+    Closure pushMethod = { configMap ->
+        UTILS.propagateStatus("Build_image", configMap["name"])
+        withCredentials([usernamePassword(credentialsId: "${DOCKER_CREDENTIALS}",
+                                          usernameVariable: 'DOCKER_USERNAME',
+                                          passwordVariable: 'DOCKER_PASSWORD')]) {
+            sh """
+                docker login ${DOCKER_REGISTRY} --username ${DOCKER_USERNAME} --password-stdin <<< \${DOCKER_PASSWORD}
+                ${CI_ROOT}/utils/docker.sh push \
+                                    --docker_registry=${DOCKER_REGISTRY} \
+                                    --name=${configMap["projectName"]} \
+                                    --version=${configMap["name"]} || return 1
+            """
+        }
+    }
+    UTILS.createStage("Push_image", pushMethod, configurationMaps)
+}
+
+def cleanup(configurationMaps) {
+    Closure cleanupMethod = { configMap ->
+        sh """
+            rm -rf ${WORKSPACE}/${BUILD_NUMBER}
+        """
+    }
+    UTILS.createStage("Cleanup", cleanupMethod, configurationMaps)
+}
+
+def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address) {
+    node(label) {
+        timeout(activity: true, time: 15) {
+            WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
+            def configurationMaps;
+            try {
+                dir ("${WORKDIR}") {
+                    cloneRepository(jenkins_github_credential_id, ngraph_git_address)
+                    // Load CI API
+                    UTILS = load "${CI_ROOT}/utils/utils.groovy"
+                    result = 'SUCCESS'
+                    // Create configuration maps
+                    configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
+                    // Build and push base images
+                    buildImage(configurationMaps)
+                    pushImage(configurationMaps)
+                }
+            }
+            catch(e) {
+                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
+                if ("$e".contains("143")) {
+                    currentBuild.result = "ABORTED"
+                } else {
+                    currentBuild.result = "FAILURE"
+                }
+            }
+            finally {
+                cleanup(configurationMaps)
+            }
+        }
+    }
+}
+
+main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS)

.ci/onnx/jenkins/dockerfiles/ubuntu-16_04.dockerfile → .ci/onnx/jenkins/dockerfiles/ubuntu_16_04.dockerfile

@@ -39,3 +39,22 @@ RUN apt-get -y install protobuf-compiler libprotobuf-dev && \
 
 # Install tox
 RUN pip3 install tox
+
+# Build nGraph master
+ARG NGRAPH_CACHE_DIR=/cache
+
+WORKDIR /root
+
+RUN git clone https://github.com/NervanaSystems/ngraph.git && \
+    cd ngraph && \
+    mkdir -p ./build && \
+    cd ./build && \
+    cmake ../ -DNGRAPH_TOOLS_ENABLE=FALSE -DNGRAPH_UNIT_TEST_ENABLE=FALSE -DNGRAPH_USE_PREBUILT_LLVM=TRUE -DNGRAPH_ONNX_IMPORT_ENABLE=TRUE && \
+    make -j $(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l)
+
+# Store built nGraph
+RUN mkdir -p ${NGRAPH_CACHE_DIR} && \
+    cp -Rf /root/ngraph/build ${NGRAPH_CACHE_DIR}/
+
+# Cleanup remaining sources
+RUN rm -rf /root/ngraph

.ci/onnx/jenkins/prepare_environment.sh

@@ -19,16 +19,7 @@
 set -x
 set -e
 
-NGRAPH_CACHE_DIR="/home"
-
-function check_cached_ngraph() {
-    set -x
-    # if no ngraph in /home - clone
-    if [ ! -e "${NGRAPH_CACHE_DIR}/ngraph" ]; then
-        cd /home/
-        git clone --single-branch https://github.com/NervanaSystems/ngraph -b master
-    fi
-}
+NGRAPH_CACHE_DIR="/cache"
 
 function build_ngraph() {
     set -x
@@ -43,21 +34,8 @@ function build_ngraph() {
                 rm -rf "${ngraph_directory}/ngraph/build"
                 rm -rf "${ngraph_directory}/ngraph_dist"
             ;;
-            UPDATE)
-                git checkout master
-                git pull origin master
-            ;;
             USE_CACHED)
-                check_cached_ngraph
-                if [[ -n $(ls /home/ngraph/build 2> /dev/null) ]]; then
-                    cp -Rf "${NGRAPH_CACHE_DIR}/ngraph/build" "${ngraph_directory}/ngraph/" || return 1
-                else
-                    return 1
-                fi
-                for f in $(find ${ngraph_directory}/ngraph/build/ -name 'CMakeCache.txt');
-                do
-                    sed -i "s\\${NGRAPH_CACHE_DIR}\\${ngraph_directory}\\g" $f
-                done
+                cp -Rf "${NGRAPH_CACHE_DIR}/build" "${ngraph_directory}/ngraph/" || return 1
             ;;
         esac
     done
@@ -76,7 +54,7 @@ function build_ngraph() {
     export PYBIND_HEADERS_PATH="${ngraph_directory}/ngraph/python/pybind11"
     export NGRAPH_CPP_BUILD_PATH="${ngraph_directory}/ngraph_dist"
     export NGRAPH_ONNX_IMPORT_ENABLE="TRUE"
-    python3 setup.py bdist_wheel
+    python3 setup.py bdist_wheel || return 1
     # Clean build artifacts
     rm -rf "${ngraph_directory}/ngraph_dist"
     return 0
@@ -87,7 +65,4 @@ mkdir -p /home/onnx_models/.onnx
 ln -s /home/onnx_models/.onnx /root/.onnx
 
 # Copy stored nGraph master and use it to build PR branch
-if ! build_ngraph "/root" "USE_CACHED"; then
-    build_ngraph "${NGRAPH_CACHE_DIR}" "UPDATE REBUILD"
-    build_ngraph "/root" "REBUILD USE_CACHED"
-fi
+build_ngraph "/root" "USE_CACHED" || build_ngraph "/root" "REBUILD"

.ci/onnx/jenkins/utils/docker.sh

@@ -15,12 +15,14 @@
 # the Materials, either expressly, by implication, inducement, estoppel or
 # otherwise. Any license under such intellectual property rights must be express
 # and approved by Intel in writing.
-readonly PARAMETERS=( 'name' 'version' 'container_name' 'volumes' 'env' 'ports' 'dockerfile_path' 'directory' 'options' 'tag' 'engine' 'frontend' 'new_tag' 'image_name' 'repository_type' 'build_cores_number')
+readonly PARAMETERS=( 'name' 'version' 'container_name' 'volumes' 'env' 'ports' 'dockerfile_path' 'directory' 'docker_registry'
+                      'options' 'tag' 'engine' 'frontend' 'new_tag' 'image_name' 'repository_type' 'build_cores_number')
 readonly WORKDIR="$(git rev-parse --show-toplevel)"
-readonly HUB_ADDRESS="hub.docker.intel.com"
 
-#Example of usage: login
+#Example of usage: login ${docker_registry}
 docker.login() {
+    local registry="${1}"
+
     local i
     local parameters
 
@@ -28,33 +30,34 @@ docker.login() {
     do
         parameters+=" --${i}"
     done
-    docker login ${parameters} ${HUB_ADDRESS}
+    docker login ${parameters} ${registry}
 }
 
-#Example of usage: get_image_name ${name} ${version} ${tag} ${engine} ${repository_type} ${frontend}
+#Example of usage: get_image_name ${docker_registry} ${name} ${version} ${tag} ${engine} ${repository_type} ${frontend}
 docker.get_image_name() {
-    local name="${1}"
-    local version="${2}"
-    local tag="${3}"
-    local engine="${4}"
-    local repository_type="${5}"
-    local frontend="${6}"
-
-    if [ "_${repository_type,,}" == "_private" ]; then
-        repository_type="_${repository_type}"
+    local registry="${1}"
+    local name="${2}"
+    local version="${3}"
+    local tag="${4}"
+    local engine="${5}"
+    local repository_type="${6}"
+    local frontend="${7}"
+
+    if [ "${repository_type,,}" == "private" ]; then
+        repository_type="${repository_type,,}/"
     else
         repository_type=""
     fi
 
     if [ ! -z ${engine} ]; then
-        engine="_${engine}"
+        engine="/${engine}"
     fi
 
     if [ ! -z ${frontend} ]; then
-        frontend="_${frontend}"
+        frontend="/${frontend}"
     fi
 
-    echo "${HUB_ADDRESS}/aibt_${name,,}${repository_type,,}/${version,,}${engine,,}${frontend,,}:${tag}"
+    echo "${registry,,}/aibt/aibt/${name,,}/${repository_type,,}${version,,}${engine,,}${frontend,,}:${tag}"
 }
 
 docker.get_git_token() {
@@ -99,7 +102,6 @@ docker.build() {
 docker.push() {
     local image_name="${1}"
 
-    docker.login
     docker push "${image_name}"
 }
 
@@ -107,7 +109,6 @@ docker.push() {
 docker.pull() {
     local image_name="${1}"
 
-    docker.login
     docker pull "${image_name}"
 }
 
@@ -269,7 +270,7 @@ main() {
         done
     done
     if [ -z ${image_name} ]; then
-        local image_name="$(docker.get_image_name ${name} ${version} ${tag:-"latest"} ${engine:-"base"} ${repository_type:-"public"} ${frontend})"
+        local image_name="$(docker.get_image_name ${docker_registry} ${name} ${version} ${tag:-"ci"} ${engine:-"base"} ${repository_type:-"public"} ${frontend})"
     fi
     case "${action}" in
         build)
@@ -299,7 +300,7 @@ main() {
         clean_up)
             docker.clean_up;;
         login)
-            docker.login;;
+            docker.login "${docker_registry}";;
         release)
             docker.release "${image_name}";;
         *)