simplify contrib/docker Makefile and make targets (#797)

* saved a simplified contrib/docker/Makefile and helper scripts

* fixed flow for make targets

* restored original definition for setting the PARALLEL option on the command line per github comments

* remove double build for make install targets

* added a save for the ngraph_dist_gcc.tgz to maintain existing behavior

* fixed passing the PARALLEL value to the make targets

* integrated latest working build-ngraph-and-test script

* integrated the latest working Makefile

* removed reference to the THIRD_PARTY_CACHE_DIR (for future)

* updated the contrib/docker/README.md file

contrib/docker/README.md

@@ -66,10 +66,22 @@ make check_gcc OS=centos74 DOCKERFILE=Dockerfile.ngraph.centos74_cmake3
 
 These helper scripts are included for use in the `Makefile` and automated (Jenkins) jobs.  **These scripts should _not_ be called directly unless you understand what they do.**
 
+#### `build-ngraph-docs.sh`
+
+A helper script to simplify implentation of the make_docs target using docker images.
+
+#### `build-ngraph-and-test.sh`
+
+A helper script to simplify implementation of make targets with multiple reference OS environments with different compilers using docker images.
+
 #### `docker_cleanup.sh`
 
 A helper script for Jenkins jobs to clean up old exited docker containers and `ngraph_*` docker images.
 
+#### `make-dimage.sh`
+
+A helper script to simplify building of docker images for multiple reference OS environments.
+
 #### `run_as_user.sh`
 
 A helper script to run as a normal user within the docker container.  This is done to avoid writing root-owned files in mounted filesystems.

contrib/docker/build-ngraph-and-test.sh

+#!  /bin/bash
+
+set -e
+# set -u  # Cannot use set -u, as activate below relies on unbound variables
+set -o pipefail
+
+# Debugging to verify builds on CentOS 7.4 and Ubuntu 16.04
+if [ -f "/etc/centos-release" ]; then
+    cat /etc/centos-release
+fi
+
+if [ -f "/etc/lsb-release" ]; then
+    cat /etc/lsb-release
+fi
+
+uname -a
+cat /etc/os-release || true
+
+echo ' '
+echo 'Contents of /home:'
+ls -la /home
+echo ' '
+echo 'Contents of /home/dockuser:'
+ls -la /home/dockuser
+echo ' '
+
+export CMAKE_OPTIONS_EXTRA=""
+
+# setting for make -j
+if [ -z ${PARALLEL} ] ; then
+    PARALLEL=22
+fi
+
+# make command to execute
+if [ -z ${CMD_TO_RUN} ] ; then
+    CMD_TO_RUN='check_gcc'
+fi
+
+# directory name to use for the build
+if [ -z ${BUILD_SUBDIR} ] ; then
+    BUILD_SUBDIR=BUILD
+fi
+
+# Option to build with GPU backend
+# default builds with CPU only
+if [ -z ${NGRAPH_GPU_ENABLE} ] ; then
+    NGRAPH_GPU_ENABLE=false
+fi
+
+# Set up the environment
+if $NGRAPH_GPU_ENABLE; then
+    export CMAKE_OPTIONS_EXTRA="-DNGRAPH_GPU_ENABLE=TRUE"
+fi
+
+export NGRAPH_REPO=/home/dockuser/ngraph-test
+
+if [ -z ${OUTPUT_DIR} ]; then
+    OUTPUT_DIR="${NGRAPH_REPO}/${BUILD_SUBDIR}"
+fi
+
+# Remove old OUTPUT_DIR directory if present for build_* targets
+if [ "$(echo ${CMD_TO_RUN} | grep build | wc -l)" != "0" ] ; then
+    ( test -d ${OUTPUT_DIR} && rm -fr ${OUTPUT_DIR} && echo "Removed old ${OUTPUT_DIR} directory" ) || echo "Previous ${OUTPUT_DIR} directory not found"
+    # Make OUTPUT_DIR directory as user
+    mkdir -p ${OUTPUT_DIR}
+    chmod ug+rwx ${OUTPUT_DIR}
+fi
+
+#TODO: add openmpi dependency to enable building with -DNGRAPH_DISTRIBUTED_ENABLE=TRUE
+#
+#if [ -z ${NGRAPH_DISTRIBUTED_ENABLE} ] ; then
+#  NGRAPH_DISTRIBUTED_ENABLE=false
+#fi
+
+#if $NGRAPH_DISTRIBUTED_ENABLE; then
+#   source /home/environment-openmpi-ci.source
+#   which mpirun
+#   mpirun --version
+#   export CMAKE_OPTIONS_EXTRA="-DNGRAPH_DISTRIBUTED_ENABLE=$NGRAPH_DISTRIBUTED_ENABLE"
+#fi
+
+GCC_VERSION=` gcc --version | grep gcc | cut -f 2 -d ')' | cut -f 2 -d ' ' | cut -f 1,2 -d '.'`
+
+if [ "${GCC_VERSION}" != "4.8" ] ; then
+    export CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA} -DNGRAPH_USE_PREBUILT_LLVM=TRUE"
+fi
+
+# Print the environment, for debugging
+echo ' '
+echo 'Environment:'
+export
+echo ' '
+
+cd $NGRAPH_REPO
+
+export CMAKE_OPTIONS_COMMON="-DNGRAPH_BUILD_DOXYGEN_DOCS=ON -DNGRAPH_BUILD_SPHINX_DOCS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo ${CMAKE_OPTIONS_EXTRA}"
+export CMAKE_OPTIONS_GCC="${CMAKE_OPTIONS_COMMON} -DNGRAPH_INSTALL_PREFIX=${NGRAPH_REPO}/BUILD-GCC/ngraph_dist"
+export CMAKE_OPTIONS_CLANG="$CMAKE_OPTIONS_COMMON -DNGRAPH_INSTALL_PREFIX=${NGRAPH_REPO}/BUILD-CLANG/ngraph_dist -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 -DNGRAPH_WARNINGS_AS_ERRORS=ON -DNGRAPH_USE_PREBUILT_LLVM=TRUE"
+
+echo "CMD_TO_RUN=${CMD_TO_RUN}"
+
+# set up the cmake environment
+if [ -z ${CMAKE_OPTIONS} ] ; then
+    if [ "$(echo ${CMD_TO_RUN} | grep gcc | wc -l)" != "0" ] ; then
+        export CMAKE_OPTIONS=${CMAKE_OPTIONS_GCC}
+    elif [ "$(echo ${CMD_TO_RUN} | grep clang | wc -l)" != "0" ] ; then
+        export CMAKE_OPTIONS=${CMAKE_OPTIONS_CLANG}
+    else
+        export CMAKE_OPTIONS=${CMAKE_OPTIONS_COMMON}
+    fi
+
+    echo "set CMAKE_OPTIONS=${CMAKE_OPTIONS}"
+fi
+
+# build and test
+export BUILD_DIR="${NGRAPH_REPO}/${BUILD_SUBDIR}"
+export GTEST_OUTPUT="xml:${BUILD_DIR}/unit-test-results.xml"
+mkdir -p ${BUILD_DIR}
+chmod ug+rwx ${BUILD_DIR}
+cd ${BUILD_DIR}
+
+echo "Build and test for ${CMD_TO_RUN} in `pwd` with specific parameters:"
+echo "    NGRAPH_REPO=${NGRAPH_REPO}"
+echo "    CMAKE_OPTIONS=${CMAKE_OPTIONS}"
+echo "    GTEST_OUTPUT=${GTEST_OUTPUT}"
+
+# only run cmake/make steps for build_* make targets
+if [ "$(echo ${CMD_TO_RUN} | grep build | wc -l)" != "0" ] ; then
+    # always run cmake/make steps
+    echo "Running cmake"
+    cmake ${CMAKE_OPTIONS} .. 2>&1 | tee ${OUTPUT_DIR}/cmake_${CMD_TO_RUN}.log
+    echo "Running make"
+    env VERBOSE=1 make -j ${PARALLEL} 2>&1 | tee ${OUTPUT_DIR}/make_${CMD_TO_RUN}.log
+    echo "CMD_TO_RUN=${CMD_TO_RUN} finished - cmake/make steps completed"
+else
+    # strip off _* from CMD_TO_RUN to pass to the ngraph make targets 
+    MAKE_CMD_TO_RUN=`echo ${CMD_TO_RUN} | sed 's/_.*//g'`
+    COMPILER=`echo ${CMD_TO_RUN} | sed 's/.*_//g'`
+
+    if [ "${MAKE_CMD_TO_RUN}" == "unit-test-check" ]; then
+    # check style before running unit tests
+        if [ -f "/usr/bin/clang-3.9" ]; then
+            echo "Running make style-check"
+            env VERBOSE=1 make -j style-check 2>&1 | tee ${OUTPUT_DIR}/make_style_check_${CMD_TO_RUN}.log
+        fi
+    fi
+
+    echo "Running make ${MAKE_CMD_TO_RUN}"
+    env VERBOSE=1 make ${MAKE_CMD_TO_RUN} 2>&1 | tee ${OUTPUT_DIR}/make_${CMD_TO_RUN}.log
+
+    if [ "${MAKE_CMD_TO_RUN}" == "install" ] ; then
+        echo "Building ngraph_dist_${COMPILER}.tgz"
+        tar czf ngraph_dist_${COMPILER}.tgz ngraph_dist 2>&1 | tee make_tarball_${COMPILER}.log
+        ls -l ngraph_dist_*.tgz
+    fi
+fi
+

contrib/docker/build-ngraph-docs.sh

+#!  /bin/bash
+
+#set -x
+set -e
+set -o pipefail
+
+
+# Debugging
+if [ -f "/etc/centos-release" ]; then
+    cat /etc/centos-release
+fi
+
+if [ -f "/etc/lsb-release" ]; then
+    cat /etc/lsb-release
+fi
+
+uname -a
+cat /etc/os-release || true
+
+echo ' '
+echo 'Contents of /home:'
+ls -la /home
+echo ' '
+echo 'Contents of /home/dockuser:'
+ls -la /home/dockuser
+echo ' '
+
+if [ -z ${CMD_TO_RUN} ] ; then
+    CMD_TO_RUN="make html"
+fi
+
+export NGRAPH_REPO=/home/dockuser/ngraph-test
+
+if [ -z ${BUILD_SUBDIR} ] ; then
+    BUILD_DIR="${NGRAPH_REPO}/doc/sphinx"
+else
+    BUILD_DIR="${NGRAPH_REPO}/${BUILD_SUBDIR}"
+fi
+
+if [ -z ${OUTPUT_DIR} ]; then
+    OUTPUT_DIR="${NGRAPH_REPO}/BUILD-DOCS"
+fi
+
+# Print the environment, for debugging
+echo ' '
+echo 'Environment:'
+export
+echo ' '
+
+# Remove old OUTPUT_DIR directory if present
+( test -d ${OUTPUT_DIR} && rm -fr ${OUTPUT_DIR} && echo "Removed old ${OUTPUT_DIR} directory" ) || echo "Previous ${OUTPUT_DIR} directory not found"
+# Make OUTPUT_DIR directory as user
+mkdir -p ${OUTPUT_DIR}
+chmod ug+rwx ${OUTPUT_DIR}
+
+# build html docs
+cd ${BUILD_DIR}
+echo "Building docs in `pwd`:"
+env VERBOSE=1 ${CMD_TO_RUN} 2>&1 | tee ${OUTPUT_DIR}/make_docs.log
+ls -l build/html/* || true
+mv build/ ${OUTPUT_DIR}

contrib/docker/make-dimage.sh

+#!  /bin/bash
+
+###
+#
+# Create a docker image that includes dependencies for building ngraph
+#
+# Uses CONTEXTDIR as the docker build context directory
+#   Default value is '.'
+#
+# Uses ./Dockerfile.${DOCKER_TAG}
+#   DOCKER_TAG is set to 'ngraph' if not set 
+#
+# Sets the docker image name as ${DOCKER_IMAGE_NAME}
+#   DOCKER_IMAGE_NAME is set to the ${DOCKER_TAG} if not set in the environment
+#   The datestamp tag is automatically appended to the DOCKER_IMAGE_NAME to create the DIMAGE_ID
+#   The ${DIMAGE_ID} docker image is created on the local server
+#   The ${DOCKER_IMAGE_NAME}:latest tag is also created by default for reference
+#
+###
+
+set -e
+#set -u
+set -o pipefail
+
+if [ -n $DOCKER_TAG ]; then
+    DOCKER_TAG=build_ngraph
+fi
+
+if [ -n $DOCKER_IMAGE_NAME ]; then
+    DOCKER_IMAGE_NAME=${DOCKER_TAG}
+fi
+
+echo "CONTEXTDIR=${CONTEXTDIR}"
+
+if [ -z ${CONTEXTDIR} ]; then
+    CONTEXTDIR='.'  # Docker image build context
+fi
+
+echo "CONTEXTDIR=${CONTEXTDIR}"
+
+if [ -n $DFILE ]; then
+    DFILE="${CONTEXTDIR}/Dockerfile.${DOCKER_TAG}"
+fi
+
+CONTEXT='.'
+
+DIMAGE_NAME="${DOCKER_IMAGE_NAME}"
+DIMAGE_VERSION=`date -Iseconds | sed -e 's/:/-/g'`
+
+DIMAGE_ID="${DIMAGE_NAME}:${DIMAGE_VERSION}"
+
+cd ${CONTEXTDIR}
+
+echo ' '
+echo "Building docker image ${DIMAGE_ID} from Dockerfile ${DFILE}, context ${CONTEXT}"
+echo ' '
+
+# build the docker base image
+docker build  --rm=true \
+       -f="${DFILE}" \
+       --build-arg http_proxy=http://proxy-us.intel.com:911 \
+       --build-arg https_proxy=https://proxy-us.intel.com:911 \
+       -t="${DIMAGE_ID}" \
+       ${CONTEXT}
+
+docker tag  "${DIMAGE_ID}"  "${DIMAGE_NAME}:latest"
+
+echo ' '
+echo 'Docker image build completed'
+echo ' '