[ONNX] ONNX CI on nGraph PRs (#1605)

* [AIBT-446] Run nGraph-ONNX tests on nGraph PRs

* Cloning nGraph from NervanaSystems

* Updated CI path to "ngraph/.ci/onnx/jenkins"

* Updated permissions

* Fixed path in find command

* Changes to notification message

* Editing CMakeCache.txt instead of removing

* Removing old ngraph wheels; cloning pybind11 if necessary

* Fixed path in removing ngraph wheel

.ci/onnx/jenkins/Jenkinsfile

+// INTEL CONFIDENTIAL
+// Copyright 2018 Intel Corporation All Rights Reserved.
+// The source code contained or described herein and all documents related to the
+// source code ("Material") are owned by Intel Corporation or its suppliers or
+// licensors. Title to the Material remains with Intel Corporation or its
+// suppliers and licensors. The Material may contain trade secrets and proprietary
+// and confidential information of Intel Corporation and its suppliers and
+// licensors, and is protected by worldwide copyright and trade secret laws and
+// treaty provisions. No part of the Material may be used, copied, reproduced,
+// modified, published, uploaded, posted, transmitted, distributed, or disclosed
+// in any way without Intel's prior express written permission.
+// No license under any patent, copyright, trade secret or other intellectual
+// property right is granted to or conferred upon you by disclosure or delivery of
+// the Materials, either expressly, by implication, inducement, estoppel or
+// otherwise. Any license under such intellectual property rights must be express
+// and approved by Intel in writing.
+
+// Set LABEL variable if empty or not declared
+try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
+// CI settings and constants
+PROJECT_NAME = "ngraph-onnx"            
+CI_ROOT = "ngraph/.ci/onnx/jenkins"
+DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci"
+NGRAPH_GIT_ADDRESS = "https://www.github.com/NervanaSystems/ngraph.git"
+ONNX_GIT_ADDRESS = "https://www.github.com/NervanaSystems/ngraph-onnx.git"
+JENKINS_GITHUB_CREDENTIAL_ID = "6887a177-8c4d-4fe9-9c3b-fcd71b22bfba"
+WARNING = ""
+
+def CloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
+    stage('Clone Repos') {
+        try {
+            checkout([$class: 'GitSCM',
+                branches: [[name: "$ghprbSourceBranch"]],
+                doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
+                userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
+                url: "${onnx_git_address}"]]])
+        }
+        catch (Exception e) {
+            WARNING = "WARNING! Failed to clone ngraph-onnx branch $ghprbSourceBranch ! Falling back to master."
+            sh """
+                echo ${WARNING}
+            """
+            checkout([$class: 'GitSCM',
+                branches: [[name: "master"]],
+                doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
+                userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
+                url: "${onnx_git_address}"]]])
+        }
+        dir ("ngraph") {
+                checkout([$class: 'GitSCM',
+                    branches: [[name: "$ghprbSourceBranch"]],
+                    doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
+                    userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
+                    url: "${ngraph_git_address}"]]])
+        }
+    }
+}
+
+def BuildImage(configurationMaps) {
+    Closure buildMethod = { configMap ->
+        sh """
+            ${CI_ROOT}/utils/docker.sh build \
+                                --name=${configMap["projectName"]} \
+                                --version=${configMap["name"]} \
+                                --dockerfile_path=${configMap["dockerfilePath"]}
+        """
+    }
+    UTILS.CreateStage("Build_Image", buildMethod, configurationMaps)
+}
+
+def RunDockerContainers(configurationMaps) {
+    Closure runContainerMethod = { configMap ->
+        UTILS.PropagateStatus("Build_Image", configMap["name"])
+        sh """
+            mkdir -p ${HOME}/ONNX_CI
+            ${CI_ROOT}/utils/docker.sh start \
+                                --name=${configMap["projectName"]} \
+                                --version=${configMap["name"]} \
+                                --container_name=${configMap["dockerContainerName"]} \
+                                --volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root"
+        """
+    }
+    UTILS.CreateStage("Run_docker_containers", runContainerMethod, configurationMaps)
+}
+
+def PrepareEnvironment(configurationMaps) {
+    Closure prepareEnvironmentMethod = { configMap ->
+        UTILS.PropagateStatus("Run_docker_containers", configMap["dockerContainerName"])
+        sh """
+            docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home
+            docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh --ngraph-branch=$ghprbSourceBranch"
+        """
+    }
+    UTILS.CreateStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
+}
+
+def RunToxTests(configurationMaps) {
+    Closure runToxTestsMethod = { configMap ->
+        UTILS.PropagateStatus("Prepare_environment", configMap["dockerContainerName"])
+        sh """
+            NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl')
+            docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/
+            mkdir -p ${HOME}/ONNX_CI/onnx_models/
+            docker cp ${configMap["dockerContainerName"]}:/root/.onnx/ ${HOME}/ONNX_CI/onnx_models/
+        """
+    }
+    UTILS.CreateStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
+}
+
+def Cleanup(configurationMaps) {
+    Closure cleanupMethod = { configMap ->
+        sh """
+            cd ${HOME}/ONNX_CI
+            ./docker.sh chmod --container_name=${configMap["dockerContainerName"]} --directory="/logs" --options="-R 777" || true
+            docker exec ${configMap["dockerContainerName"]} rm -rf /root/* || true
+            ./docker.sh stop --container_name=${configMap["dockerContainerName"]} || true
+            ./docker.sh remove --container_name=${configMap["dockerContainerName"]} || true
+            ./docker.sh clean_up || true
+            rm ${HOME}/ONNX_CI/docker.sh
+            rm -rf ${WORKSPACE}/${BUILD_NUMBER}
+        """
+    }
+    UTILS.CreateStage("Cleanup", cleanupMethod, configurationMaps)
+}
+
+def Notify() {
+    configurationMaps = []
+    configurationMaps.add([
+        "name": "notify"
+    ])
+    String notifyPeople = "$ghprbPullAuthorEmail, $ghprbActualCommitAuthorEmail"
+    Closure notifyMethod = { configMap ->
+        if(currentBuild.result != "FAILURE") {
+            currentBuild.result = "SUCCESS"
+        }
+        if(currentBuild.result == "FAILURE" || WARNING != "") {
+            emailext (
+                subject: "NGraph-Onnx CI: NGraph PR $ghprbPullId $currentBuild.result!",
+                body: """
+                    <table style="width:100%">
+                        <tr><td>${WARNING}</td></tr>
+                        <tr><td>Status:</td> <td>${currentBuild.result}</td></tr>
+                        <tr><td>Repository</td> <td>$ghprbGhRepository</td></tr>
+                        <tr><td>Branch:</td> <td>$ghprbSourceBranch</td></tr>
+                        <tr><td>Pull Request:</td> <td>$ghprbPullId</td></tr>
+                        <tr><td>Commit SHA:</td> <td>$ghprbActualCommit</td></tr>
+                        <tr><td>Link:</td> <td>$ghprbPullLink</td></tr>
+                    </table>
+                """,
+                to: "${notifyPeople}"
+            )
+        }
+    }
+    UTILS.CreateStage("Notify", notifyMethod, configurationMaps)
+}
+
+def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
+    node(label) {
+        timeout(activity: true, time: 60) {        
+            WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
+            def configurationMaps;
+            try {
+                dir ("${WORKDIR}") {
+                    CloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
+                    // Load CI API
+                    UTILS = load "${CI_ROOT}/utils/utils.groovy"
+                    result = 'SUCCESS'
+                    // Create configuration maps
+                    configurationMaps = UTILS.GetDockerEnvList(projectName, dockerContainerName, projectRoot)
+                    // Execute CI steps
+                    BuildImage(configurationMaps)
+                    RunDockerContainers(configurationMaps)
+                    PrepareEnvironment(configurationMaps)
+                    RunToxTests(configurationMaps)
+                }
+            }
+            finally {
+                Cleanup(configurationMaps)
+                Notify()
+            }
+        }
+    }
+}
+
+main(LABEL, PROJECT_NAME, CI_ROOT, DOCKER_CONTAINER_NAME, JENKINS_GITHUB_CREDENTIAL_ID, NGRAPH_GIT_ADDRESS, ONNX_GIT_ADDRESS)

.ci/onnx/jenkins/README.md

+# MANUAL REPRODUCTION INSTRUCTION
+From directory containing CI scripts execute runCI.sh bash script:
+
+```
+cd <path-to-repo>/.ci/jenkins/
+./runCI.sh
+```
+
+To remove all items created during script execution (files, directories, docker images and containers), run:
+
+```
+./runCI.sh --cleanup
+```

.ci/onnx/jenkins/dockerfiles/ubuntu-16_04.dockerfile

+FROM ubuntu:16.04
+
+ARG HOME=/root
+ARG http_proxy
+ARG https_proxy
+ENV http_proxy ${http_proxy}
+ENV https_proxy ${https_proxy}
+
+# nGraph dependencies
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  cmake \
+  clang-3.9 \
+  git \
+  curl \
+  zlib1g \
+  zlib1g-dev \
+  libtinfo-dev \
+  unzip \
+  autoconf \
+  automake \
+  libtool && \
+  apt-get clean autoclean && apt-get autoremove -y
+
+# Python dependencies
+RUN apt-get -y install python3 \
+                       python3-pip \
+                       python3-dev \
+                       python-virtualenv && \
+    apt-get clean autoclean && \
+    apt-get autoremove -y
+
+RUN pip3 install --upgrade pip setuptools wheel
+
+# ONNX dependencies
+RUN apt-get -y install protobuf-compiler libprotobuf-dev && \
+    apt-get clean autoclean && \
+    apt-get autoremove -y
+
+# Install tox
+RUN pip3 install tox

.ci/onnx/jenkins/prepare_environment.sh

+#!/bin/bash
+# INTEL CONFIDENTIAL
+# Copyright 2018 Intel Corporation All Rights Reserved.
+# The source code contained or described herein and all documents related to the
+# source code ("Material") are owned by Intel Corporation or its suppliers or
+# licensors. Title to the Material remains with Intel Corporation or its
+# suppliers and licensors. The Material may contain trade secrets and proprietary
+# and confidential information of Intel Corporation and its suppliers and
+# licensors, and is protected by worldwide copyright and trade secret laws and
+# treaty provisions. No part of the Material may be used, copied, reproduced,
+# modified, published, uploaded, posted, transmitted, distributed, or disclosed
+# in any way without Intel's prior express written permission.
+# No license under any patent, copyright, trade secret or other intellectual
+# property right is granted to or conferred upon you by disclosure or delivery of
+# the Materials, either expressly, by implication, inducement, estoppel or
+# otherwise. Any license under such intellectual property rights must be express
+# and approved by Intel in writing.
+
+
+PATTERN='[-a-zA-Z0-9_]*='
+for i in "$@"
+do
+    case $i in
+        --help*)
+            printf "Following parameters are available:
+    
+            --help  displays this message
+            --ngraph-branch ngraph branch name to build
+            "
+            exit 0
+        ;;
+        --ngraph-branch=*)
+            NGRAPH_BRANCH=`echo $i | sed "s/${PATTERN}//"`
+        ;;
+    esac
+done
+
+set -x
+
+function build_ngraph() {
+    # directory containing ngraph repo
+    local ngraph_directory="$1"
+    cd "${ngraph_directory}/ngraph"
+    mkdir -p ./build
+    cd ./build
+    cmake ../ -DNGRAPH_USE_PREBUILT_LLVM=TRUE -DNGRAPH_ONNX_IMPORT_ENABLE=TRUE -DCMAKE_INSTALL_PREFIX="${ngraph_directory}/ngraph_dist"
+    rm "${ngraph_directory}"/ngraph/python/dist/ngraph*.whl
+    make -j $(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l)
+    make install
+    cd "${ngraph_directory}/ngraph/python"
+    if [ ! -d ./pybind11 ]; then
+        git clone --recursive -b allow-nonconstructible-holders https://github.com/jagerman/pybind11.git
+    fi
+    export PYBIND_HEADERS_PATH="${ngraph_directory}/ngraph/python/pybind11"
+    export NGRAPH_CPP_BUILD_PATH="${ngraph_directory}/ngraph_dist"
+    python3 setup.py bdist_wheel
+}
+
+# Clone and build nGraph master 
+cd /home
+if [ -e ./ngraph ]; then
+    cd ./ngraph
+    if [[ $(git pull) != *"Already up-to-date"* ]]; then
+        build_ngraph "/home"
+    fi
+else
+    git clone https://github.com/NervanaSystems/ngraph.git -b master
+    build_ngraph "/home"
+fi
+
+cp -R /home/ngraph/build /root/ngraph/
+cp -R /home/ngraph_dist /root/
+# Change directory to ngraph cloned initially by CI, which is already on relevant branch
+cd /root/ngraph
+for f in $(find build/ -name 'CMakeCache.txt'); 
+do 
+    sed -i 's/home/root/g' $f
+done
+build_ngraph "/root"
+
+# Copy Onnx models
+if [ -d /home/onnx_models/.onnx ]; then
+    rsync -avhz /home/onnx_models/.onnx /root/ngraph-onnx/
+fi

.ci/onnx/jenkins/runCI.sh

+#!/bin/bash
+
+# ******************************************************************************
+# Copyright 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ******************************************************************************
+
+CI_PATH="$(pwd)"
+CI_ROOT=".ci/jenkins"
+REPO_ROOT="${CI_PATH%$CI_ROOT}"
+DOCKER_CONTAINER="ngraph-onnx_ci"
+
+# Function run() builds image with requirements needed to build ngraph and run onnx tests, runs container and executes tox tests
+function run() {
+    cd ./dockerfiles
+    docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -f=./ubuntu-16_04.dockerfile -t ngraph-onnx:ubuntu-16_04 .
+
+    cd "${CI_PATH}"
+    mkdir -p ${HOME}/ONNX_CI
+    if [[ -z $(docker ps -a | grep -i "${DOCKER_CONTAINER}") ]]; 
+    then 
+        docker run -h "$(hostname)" --privileged --name "${DOCKER_CONTAINER}" -v ${HOME}/ONNX_CI:/home -v "${REPO_ROOT}":/root -d ngraph-onnx:ubuntu-16_04 tail -f /dev/null
+        docker cp ./prepare_environment.sh "${DOCKER_CONTAINER}":/home
+        docker exec "${DOCKER_CONTAINER}" ./home/prepare_environment.sh
+    fi
+
+    NGRAPH_WHL=$(docker exec ${DOCKER_CONTAINER} find /home/ngraph/python/dist/ -name "ngraph*.whl")
+    docker exec -e TOX_INSTALL_NGRAPH_FROM="${NGRAPH_WHL}" "${DOCKER_CONTAINER}" tox -c /root
+
+    echo "========== FOLLOWING ITEMS WERE CREATED DURING SCRIPT EXECUTION =========="
+    echo "Docker image: ngraph-onnx:ubuntu-16_04"
+    echo "Docker container: ${DOCKER_CONTAINER}"
+    echo "Directory: ${HOME}/ONNX_CI"
+    echo "Multiple files generated during tox execution"
+    echo ""
+    echo "TO REMOVE THEM RUN THIS SCRIPT WITH PARAMETER: --cleanup"
+}
+
+# Function cleanup() removes items created during script execution
+function cleanup() {
+    docker exec "${DOCKER_CONTAINER}" bash -c 'rm -rf /home/$(find /home/ -user root)'
+    rm -rf ${HOME}/ONNX_CI
+    docker exec "${DOCKER_CONTAINER}" bash -c 'rm -rf /root/$(find /root/ -user root)'
+    docker rm -f "${DOCKER_CONTAINER}"
+    docker rmi --force ngraph-onnx:ubuntu-16_04
+}
+
+PATTERN='[-a-zA-Z0-9_]*='
+for i in "$@"
+do
+    case $i in
+        --help*)
+            printf "Following parameters are available:
+    
+            --help  displays this message
+            --cleanup  removes docker image, container and files created during script execution
+            "
+            exit 0
+        ;;
+        --cleanup*)
+            cleanup
+            exit 0
+        ;;
+    esac
+done
+
+set -x
+
+run
\ No newline at end of file

.ci/onnx/jenkins/utils/docker.sh

+#!/bin/bash
+# INTEL CONFIDENTIAL
+# Copyright 2018 Intel Corporation All Rights Reserved.
+# The source code contained or described herein and all documents related to the
+# source code ("Material") are owned by Intel Corporation or its suppliers or
+# licensors. Title to the Material remains with Intel Corporation or its
+# suppliers and licensors. The Material may contain trade secrets and proprietary
+# and confidential information of Intel Corporation and its suppliers and
+# licensors, and is protected by worldwide copyright and trade secret laws and
+# treaty provisions. No part of the Material may be used, copied, reproduced,
+# modified, published, uploaded, posted, transmitted, distributed, or disclosed
+# in any way without Intel's prior express written permission.
+# No license under any patent, copyright, trade secret or other intellectual
+# property right is granted to or conferred upon you by disclosure or delivery of
+# the Materials, either expressly, by implication, inducement, estoppel or
+# otherwise. Any license under such intellectual property rights must be express
+# and approved by Intel in writing.
+readonly PARAMETERS=( 'name' 'version' 'container_name' 'volumes' 'env' 'ports' 'dockerfile_path' 'directory' 'options' 'tag' 'engine' 'frontend' 'new_tag' 'image_name' 'repository_type' 'build_cores_number')
+readonly WORKDIR="$(git rev-parse --show-toplevel)"
+readonly HUB_ADDRESS="hub.docker.intel.com"
+
+#Example of usage: login
+docker.login() {
+    local i
+    local parameters
+
+    for i in $(cat ${HOME}/tokens/docker)
+    do
+        parameters+=" --${i}"
+    done
+    docker login ${parameters} ${HUB_ADDRESS}
+}
+
+#Example of usage: get_image_name ${name} ${version} ${tag} ${engine} ${repository_type} ${frontend}
+docker.get_image_name() {
+    local name="${1}"
+    local version="${2}"
+    local tag="${3}"
+    local engine="${4}"
+    local repository_type="${5}"
+    local frontend="${6}"
+
+    if [ "_${repository_type,,}" == "_private" ]; then
+        repository_type="_${repository_type}"
+    else
+        repository_type=""
+    fi
+
+    if [ ! -z ${engine} ]; then
+        engine="_${engine}"
+    fi
+
+    if [ ! -z ${frontend} ]; then
+        frontend="_${frontend}"
+    fi
+
+    echo "${HUB_ADDRESS}/aibt_${name,,}${repository_type,,}/${version,,}${engine,,}${frontend,,}:${tag}"
+}
+
+docker.get_git_token() {
+    local token=$(cat ${HOME}/tokens/private_git)
+    echo "${token}"
+}
+
+#Example of usage: build ${image_name} ${dockerfile}
+docker.build() {
+    local image_name="${1}"
+    local dockerfile_path="${2}"
+    local repository_type="${3}"
+    local build_cores_number="${4}"
+
+    if [ ${repository_type} == "private" ]; then
+        BUILD_ARGS="--build-arg REPOSITORY_TYPE=private --build-arg TOKEN=$(docker.get_git_token)"
+    fi
+
+    # Add http_proxy if exists
+    if [ -n ${http_proxy} ]; then
+        BUILD_ARGS+="--build-arg http_proxy=${http_proxy} "
+    fi
+
+    # Add https_proxy if exists
+    if [ -n ${https_proxy} ]; then
+        BUILD_ARGS+="--build-arg https_proxy=${https_proxy} "
+    fi
+
+    # If build_cores_number was not passed - detect number of build cores
+    if [ -z ${build_cores_number} ]; then
+        BUILD_ARGS+="--build-arg BUILD_CORES_NUMBER=$(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l) "
+    fi
+
+    docker build ${BUILD_ARGS} -f "${WORKDIR}/${dockerfile_path}" -t "${image_name}" .
+    local exit_code=${?}
+    if [ ${exit_code} != "0" ]; then
+        exit ${exit_code}
+    fi
+}
+
+#Example of usage: push ${image_name}
+docker.push() {
+    local image_name="${1}"
+
+    docker.login
+    docker push "${image_name}"
+}
+
+#Example of usage: pull ${image_name}
+docker.pull() {
+    local image_name="${1}"
+
+    docker.login
+    docker pull "${image_name}"
+}
+
+#Example of usage: shell ${image_name} ${container_name} ${volumes} ${env} ${ports}
+docker.shell() {
+    local image_name=${1}
+    local container_name="${2}"
+    local volumes="${3}"
+    local env="${4}"
+    local ports="${5}"
+
+    docker run -h "$(hostname)" --rm --privileged --name "${container_name}" -i -t "${ports}" "${volumes}" "${env}" \
+        "${image_name}" /bin/bash
+}
+
+#Example of usage: run ${image_name} ${container_name} ${volumes} ${env} ${ports}
+docker.run() {
+    local image_name="${1}"
+    local container_name="${2}"
+    local volumes="${3}"
+    local env="${4}"
+    local ports="${5}"
+    local engine="${6}"
+
+    DOCKER_COMMAND="docker"
+
+    if [ ${engine,,} == "cudnn" ]; then
+        DOCKER_COMMAND="nvidia-docker"
+    fi
+
+    ${DOCKER_COMMAND} run -h "$(hostname)" --rm --privileged --name "${container_name}" "${ports}" "${volumes}" "${env}" "${image_name}"
+}
+
+#Example of usage: start ${image_name} ${container_name} ${volumes} ${env} ${ports}
+docker.start() {
+    local image_name="${1}"
+    local container_name="${2}"
+    local volumes="${3}"
+    local env="${4}"
+    local ports="${5}"
+    local engine="${6}"
+
+    docker ps -a | grep "${container_name}"  &> /dev/null
+    if [ $? == 0 ]; then
+        docker.stop "${container_name}"
+        docker.remove "${container_name}"
+    fi
+
+    DOCKER_COMMAND="docker"
+
+    if [ ${engine,,} == "cudnn" ]; then
+        DOCKER_COMMAND="nvidia-docker"
+    fi
+
+    CMD="${DOCKER_COMMAND} run -h "$(hostname)" -id --privileged --name "${container_name}" "${ports}" "${volumes}" "${env}" "${image_name}" tail -f /dev/null"
+
+    eval "${CMD}"
+}
+
+#Example of usage: commit ${image_name} ${container_name}
+docker.commit() {
+    local image_name="${1}"
+    local container_name="${2}"
+
+    docker commit "${container_name}" "${image_name}"
+}
+
+#Example of usage: tag ${image_name} ${new_tag}
+docker.tag() {
+    local image_name="${1}"
+    local new_tag="${2}"
+
+    docker tag "${image_name}" "${image_name/:*/:${new_tag}}"
+}
+
+#Example of usage: release ${image_name}
+docker.release() {
+    local image_name="${1}"
+
+    docker.tag "${image_name}" "latest"
+}
+
+#Example of usage: stop ${container_name}
+docker.stop() {
+    local container_name="${1}"
+
+    docker stop "${container_name}" || true
+}
+
+#Example of usage: clean_workdir ${container_name} ${directory} ${options}
+docker.chmod() {
+    local container_name="${1}"
+    local directory="${2}"
+    local options="${3}"
+
+    docker start ${container_name}
+    docker exec ${container_name} bash -c "cd ${directory}; chmod ${options} \$(ls -a | tail -n +3)"
+}
+
+#Example of usage: remove ${container_name}
+docker.remove() {
+    local container_name="${1}"
+
+    docker rm "${container_name}" || true
+}
+
+#Example of usage: prune
+docker.format() {
+    docker system prune --all --force
+}
+
+#Example of usage: clean_up
+docker.clean_up() {
+    #list of all container
+    local -r containers_list="$(docker ps -a -q)"
+    #list of wrongly taged images
+    local -r images_list="$(docker images --format "{{.Repository}}:{{.Tag}}->{{.ID}}" | grep '<none>')"
+
+    if [[ ! -z "${containers_list}" ]]; then
+        #Stop containers
+        docker stop ${containers_list}
+        #Remove containers
+        docker rm ${containers_list}
+    fi
+    if [[ ! -z "${images_list}" ]]; then
+        # Delete images
+        for image in ${images_list}
+        do
+            docker rmi ${image/->*/} #remove by name
+            docker rmi ${image/*->/} #remove by id
+        done
+    fi
+    #Clean docker system
+    printf 'y' | docker system prune
+}
+
+#Script help
+usage() {
+    cat <<EOF
+    Usage: $0 [options]
+EOF
+}
+
+main() {
+    local pattern='[-a-zA-Z0-9_]*='
+    local i
+    local action=${1}; shift #assign first argument and remove from the argument list
+
+    #parse arguments
+    for i in "${@}"
+    do
+        local parameter_name
+        for parameter_name in "${PARAMETERS[@]}"
+        do
+            if [[ ${i} == "--${parameter_name}="* ]]; then
+                local value="${i//${pattern}/}"
+                eval "local ${parameter_name}=\"${value}\""
+            fi
+        done
+    done
+    if [ -z ${image_name} ]; then
+        local image_name="$(docker.get_image_name ${name} ${version} ${tag:-"latest"} ${engine:-"base"} ${repository_type:-"public"} ${frontend})"
+    fi
+    case "${action}" in
+        build)
+            docker.build "${image_name}" "${dockerfile_path}" "${repository_type:-"public"}" "${build_cores_number}";;
+        push)
+            docker.push "${image_name}";;
+        pull)
+            docker.pull "${image_name}";;
+        shell)
+            docker.shell "${image_name}" "${container_name}" "${volumes}" "${env}" "${ports}";;
+        run)
+            docker.run "${image_name}" "${container_name}" "${volumes}" "${env}" "${ports}" "${engine:-"base"}";;
+        start)
+            docker.start "${image_name}" "${container_name}" "${volumes}" "${env}" "${ports}" "${engine:-"base"}";;
+        commit)
+            docker.commit "${image_name}" "${container_name}";;
+        tag)
+            docker.tag "${image_name}" "${new_tag}";;
+        stop)
+            docker.stop "${container_name}";;
+        remove)
+            docker.remove "${container_name}";;
+        chmod)
+            docker.chmod "${container_name}" "${directory}" "${options}";;
+        format)
+            docker.format;;
+        clean_up)
+            docker.clean_up;;
+        login)
+            docker.login;;
+        release)
+            docker.release "${image_name}";;
+        *)
+            usage;;
+    esac
+}
+
+if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then
+    main "${@}"
+fi

.ci/onnx/jenkins/utils/utils.groovy

+// INTEL CONFIDENTIAL
+// Copyright 2018 Intel Corporation All Rights Reserved.
+// The source code contained or described herein and all documents related to the
+// source code ("Material") are owned by Intel Corporation or its suppliers or
+// licensors. Title to the Material remains with Intel Corporation or its
+// suppliers and licensors. The Material may contain trade secrets and proprietary
+// and confidential information of Intel Corporation and its suppliers and
+// licensors, and is protected by worldwide copyright and trade secret laws and
+// treaty provisions. No part of the Material may be used, copied, reproduced,
+// modified, published, uploaded, posted, transmitted, distributed, or disclosed
+// in any way without Intel's prior express written permission.
+// No license under any patent, copyright, trade secret or other intellectual
+// property right is granted to or conferred upon you by disclosure or delivery of
+// the Materials, either expressly, by implication, inducement, estoppel or
+// otherwise. Any license under such intellectual property rights must be express
+// and approved by Intel in writing.
+STAGES_STATUS_MAP = [:]
+
+def GetDockerEnvList(String projectName, String dockerContainerNamePrefix, String projectRoot = projectName) {
+    /**
+    * This method generates configuration map list using dockerfiles available in dockerfiles directory
+    *
+    * @param projectName name of the project used in paths and configuration map.
+    * @param dockerContainerNamePrefix docker container name prefix.
+    * @param projectRoot path to project root containing directory with dockerfiles to run
+    */
+
+    def rawList = findFiles(glob: "${projectRoot}/dockerfiles/*.dockerfile")
+    def envList = []
+    for (int i = 0; i < rawList.size(); ++i) {
+        def name = rawList[i].name - '.dockerfile'
+        def dockerContainerName = "${dockerContainerNamePrefix}_${name}"
+        envList.add([name:name, // name is the only obligatory vaiable
+                     dockerfilePath:rawList[i].path,
+                     projectName:projectName,
+                     dockerContainerName:dockerContainerName])
+    }
+    return envList
+}
+
+def GenerateMap(Closure method, configurationMaps) {
+    /**
+    * Generates map for method using configurationMaps.
+    *
+    * @param method Method that will be executed in each map(configuration).
+    * @param configurationMaps Map of configuration that will be parallelized.
+    */
+
+    def executionMap = [:]
+    for (int i = 0; i < configurationMaps.size(); ++i) {
+        configMap = configurationMaps[i]
+        executionMap[configMap["name"]] = {
+            method(configMap)
+        }
+    }
+    return executionMap
+}
+
+def CreateStage(String stageName, Closure method, configurationMaps) {
+    /**
+    * Create pipeline stage.
+    * 
+    * @param stageName Name of stage that will be create.
+    * @param method Method that will be executed in each map(configuration).
+    * @param configurationMaps Map of configuration that will be parallelized.
+    */
+
+    stage(stageName) {
+        // Add current stage name to configurationMaps
+        for (int i = 0; i < configurationMaps.size(); ++i) {
+            configurationMaps[i]["stageName"] = stageName
+        }
+
+        Closure genericBodyMethod = { configMap ->
+            def status = "SUCCESS"
+            try {
+                method(configMap)
+            } catch(e) {
+                status = "FAILURE"
+                throw e
+            } finally {
+                UTILS.SetConfigurationStatus(configMap["stageName"], configMap["name"], status)
+            }
+        }
+
+        try {
+            def prepareEnvMap = GenerateMap(genericBodyMethod, configurationMaps)
+            parallel prepareEnvMap
+        } catch(e) {
+            Exception(e)
+        }
+    }
+}
+
+def SetConfigurationStatus(String stageName, String configurationName, String status) {
+    /**
+    * Set stage status.
+    * 
+    * @param stageName The name of the stage in which the configuration is.
+    * @param configurationName The name of the configuration whose status will be updated.
+    * @param status Configuration status: SUCCESS or FAILURE.
+    */
+    if (!STAGES_STATUS_MAP.containsKey(stageName)) {
+        STAGES_STATUS_MAP[stageName] = [:]
+    }
+    if (["FAILURE", "SUCCESS"].contains(status.toUpperCase())) {
+        STAGES_STATUS_MAP[stageName][configurationName] = status.toUpperCase()
+    } else {
+        throw new Exception("Not supported status name.")
+    }
+}
+
+def PropagateStatus(String parentStageName, String parentConfigurationName) {
+    /**
+    * Popagate status in parent configuration fails.
+    * This method will throw exeption "Propagating status of $parentStageName"
+    * if parent configuration name status is FAILURE
+    * 
+    * @param parentStageName The name of the stage in which the configuration is.
+    * @param parentConfigurationName The name of the configuration whose status will be propagated.
+    */
+
+    parentStageStatus = STAGES_STATUS_MAP[parentStageName][parentConfigurationName]
+    if (parentStageStatus == "FAILURE") {
+        throw new Exception("Propagating status of ${parentStageName}")
+    }
+}
+
+def ShowStatusMap() {
+    /**
+    * Display status map for every defined stage.
+    */
+
+    echo "${STAGES_STATUS_MAP}"
+}
+
+def Exception(e) {
+    currentBuild.result = 'FAILURE'
+}
+
+return this