Unverified Commit 480cf8f3 authored by DawnStone's avatar DawnStone Committed by GitHub

enable dockerfile build/test with CentOS 7.4 (#517)

* added Dockerfile.ngraph_cpp.centos74_cmake3, run_as_centos_user.sh, and run_as_ubuntu_user.sh to enable build/test with centos74

* added RUN_AS_USER variable as input for contrib/docker/Makefile

* moved dockerfile for centos7 to contrib/docker

* updated Makefile to take OS as a parameter from the command line for contrib/docker make targets

* fixed capability for setting OS variable on the command line for the contrib/docker/Makefile

* setting the NGRAPH_USE_PREBUILT_LLVM only if the gcc version is not 4.8

* setting NGRAPH_USE_PREBUILT_LLVM for all ubuntu builds

* updated contrib/docker/Makefile to enable the OS=centos74 override for make check_gcc
parent e1818685
# Environment to build and unit-test private-ngraph-cpp on centos74
# with gcc 4.8.5
# with python 2.7
# with cmake3
# LLVM/clang will be built from source
FROM centos:7
# Added install for perl Data::Dumper to avoid a compile error
RUN yum -y update && \
yum -y --enablerepo=extras install epel-release && \
yum -y install \
gcc gcc-c++ \
cmake3 make \
git \
wget patch diffutils zlib-devel ncurses-devel libtinfo-dev \
python python-devel python-setuptools \
doxygen \
which \
'perl(Data::Dumper)'
RUN ln -s /usr/bin/cmake3 /usr/bin/cmake
RUN cmake --version
RUN make --version
RUN gcc --version
RUN c++ --version
RUN easy_install pip
RUN pip install virtualenv
# Install some pip packages
RUN pip install numpy
# need to use sphinx version 1.6 to build docs
# installing with apt-get install python-sphinx installs sphinx version 1.3.6 only
# added install for python-pip above and
# installed sphinx with pip to get the updated version 1.6.5
# allows for make html build under the doc/source directory as an interim build process
RUN pip install sphinx
# breathe package required to build documentation
RUN pip install breathe
WORKDIR /home
# Environment to build and unit-test private-ngraph-cpp
FROM ubuntu:16.04
RUN apt-get update && apt-get install -y \
build-essential cmake \
clang-3.9 clang-format-3.9 \
git \
wget patch diffutils zlib1g-dev libtinfo-dev \
doxygen python-pip
RUN apt-get clean autoclean && \
apt-get autoremove -y
RUN pip install --upgrade pip
# need to use sphinx version 1.6 to build docs
# installing with apt-get install python-sphinx installs sphinx version 1.3.6 only
# added install for python-pip above and
# installed sphinx with pip to get the updated version 1.6.5
# allows for make html build under the doc/source directory as an interim build process
RUN pip install sphinx
RUN pip install breathe
WORKDIR /home
# Environment to build and unit-test private-ngraph-cpp
FROM ubuntu:16.04
RUN apt-get update && apt-get install -y \
build-essential cmake \
git \
wget patch diffutils zlib1g-dev libtinfo-dev \
doxygen python-pip
RUN which gcc && gcc --version || true
RUN which cc++ && cc++ --version || true
RUN apt-get install -y gcc-4.8 gcc++-4.8
RUN which gcc && gcc --version || true
RUN which cc++ && cc++ --version || true
RUN ln -s /usr/bin/gcc-4.8 /usr/bin/gcc || true
RUN ln -s /usr/bin/cc++-4.8 /usr/bin/cc++ || true
RUN which gcc && gcc --version || true
RUN which cc++ && cc++ --version || true
#RUN apt-get clean autoclean && \
# apt-get autoremove -y
RUN pip install --upgrade pip
# need to use sphinx version 1.6 to build docs
# installing with apt-get install python-sphinx installs sphinx version 1.3.6 only
# added install for python-pip above and
# installed sphinx with pip to get the updated version 1.6.5
# allows for make html build under the doc/source directory as an interim build process
RUN pip install sphinx
# breathe package required to build documentation
RUN pip install breathe
WORKDIR /home
# Basic Makefile for contrib/docker. This can be expanded later as more targets
# are added.
# Default is to build with -j for parallel builds. Turn off with
# make PARELLEL=
# Default is to build with -j for parallel cmake/make. Turn off with
# make PARALLEL=
PARALLEL=-j
# DIR is an internal variable that serves as an anchor to this cloned git
......@@ -28,13 +28,27 @@ DBUILD_DIR = ${DIR}/contrib/docker/.build-${DBUILD_VERSION}
# Enable additional options to be added on the command line
ifndef CMAKE_OPTIONS_EXTRA
CMAKE_OPTIONS_EXTRA = ""
CMAKE_OPTIONS_EXTRA=
endif
# OS set to 'ubuntu1604' by default
# can be overridden on the command line with 'make <target> OS=centos74"
ifndef OS
OS="ubuntu1604"
endif
ifeq ("$(shell echo ${OS} | grep centos)","centos74")
RUN_AS_USER_SCRIPT=${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_centos_user.sh
DOCKERFILE=Dockerfile.ngraph_cpp.centos74_cmake3
else
DOCKERFILE ?= "Dockerfile.ngraph_cpp"
RUN_AS_USER_SCRIPT ?= ${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_ubuntu_user.sh
endif
# For gcc builds, we do NOT regard warnings as errors
# For clang builds, we DO make warnings into errors
CMAKE_OPTIONS_COMMON=-DNGRAPH_BUILD_DOXYGEN_DOCS=ON -DNGRAPH_BUILD_SPHINX_DOCS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo $(CMAKE_OPTIONS_EXTRA)
CMAKE_OPTIONS_GCC=$(CMAKE_OPTIONS_COMMON) -DNGRAPH_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC/ngraph_dist -DNGRAPH_USE_PREBUILT_LLVM=TRUE
CMAKE_OPTIONS_GCC=$(CMAKE_OPTIONS_COMMON) -DNGRAPH_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC/ngraph_dist
CMAKE_OPTIONS_CLANG=$(MAKE_OPTIONS_COMMON)-DNGRAPH_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG/ngraph_dist -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 -DNGRAPH_WARNINGS_AS_ERRORS=ON -DNGRAPH_USE_PREBUILT_LLVM=TRUE
CALLER_UID := $(shell id -u)
......@@ -66,9 +80,12 @@ all: check_gcc check_clang
# Docker actions
expand_dockerfile_templates:
@echo "OS=${OS}"
@echo "DOCKERFILE=${DOCKERFILE}"
@echo "RUN_AS_USER_SCRIPT=${RUN_AS_USER_SCRIPT}"
cd "${DIR}"/contrib/docker
mkdir "${DBUILD_DIR}" || true
sed -e 's/\(FROM ngraph.*\)/\1:${DBUILD_VERSION}/' Dockerfile.ngraph_cpp > "${DBUILD_DIR}"/Dockerfile.build_ngraph_cpp
sed -e 's/\(FROM ngraph.*\)/\1:${DBUILD_VERSION}/' ${DOCKERFILE} > "${DBUILD_DIR}"/Dockerfile.build_ngraph_cpp
build_docker_image: expand_dockerfile_templates
$(DOCKER_BUILD) -f="${DBUILD_DIR}"/Dockerfile.build_ngraph_cpp --build-arg python_version="${PYTHON_VERSION}" -t=build_ngraph_cpp:"${DBUILD_VERSION}" .
......@@ -85,11 +102,11 @@ sphinx_doc: build_docker_image
# sphinx html docs build
docker run --rm --tty \
${VOLUME} \
${DOCKER_RUN_ENV} \
{DOCKER_RUN_ENV} \
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/doc/sphinx; env VERBOSE=1 make html 2>&1 | tee make_sphinx_html.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
# Build
build_all: build_gcc build_clang
......@@ -105,11 +122,11 @@ build_gcc: build_docker_image
${DOCKER_RUN_ENV} \
--env GTEST_OUTPUT="xml:${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC/unit-test-results.xml" \
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC; cmake ${CMAKE_OPTIONS_GCC} .. 2>&1 | tee cmake_gcc.log ; env VERBOSE=1 make ${PARALLEL} 2>&1 | tee make_gcc.log" \
--env RUN_CMD="set -x; set -e ; set -o pipefail ; if [ -f "/etc/centos-release" ]; then cat /etc/centos-release; fi; if [ -f "/etc/lsb-release" ]; then cat /etc/lsb-release; fi; uname -a ; cat /etc/os-release || true; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC; cmake ${CMAKE_OPTIONS_GCC} .. 2>&1 | tee cmake_gcc.log ; env VERBOSE=1 make ${PARALLEL} 2>&1 | tee make_gcc.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
build_clang:
build_clang: build_docker_image
# Remove old distribution directory if present
( test -d "${DIR}"/BUILD-CLANG/ngraph_dist && rm -fr "${DIR}"/BUILD-CLANG/ngraph_dist && echo "Removed old ${DIR}/BUILD-CLANG/ngraph_dist directory" ) || echo "Previous ngraph_dist directory not found"
# Make BUILD-CLANG directory as user
......@@ -120,9 +137,9 @@ build_clang:
${DOCKER_RUN_ENV} \
--env GTEST_OUTPUT="xml:${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG/unit-test-results.xml" \
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG; cmake ${CMAKE_OPTIONS_CLANG} .. 2>&1 | tee cmake_clang.log ; env VERBOSE=1 make ${PARALLEL} 2>&1 | tee make_clang.log" \
--env RUN_CMD="set -e ; set -o pipefail ; if [ -f "/etc/centos-release" ]; then cat /etc/centos-release; fi; if [ -f "/etc/lsb-release" ]; then cat /etc/lsb-release; fi; uname -a ; cat /etc/os-release || true; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG; cmake ${CMAKE_OPTIONS_CLANG} .. 2>&1 | tee cmake_clang.log ; env VERBOSE=1 make ${PARALLEL} 2>&1 | tee make_clang.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
# Check (run unit-tests)
......@@ -134,9 +151,9 @@ check_gcc: build_gcc
${DOCKER_RUN_ENV} \
--env GTEST_OUTPUT="xml:${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC/unit-test-results.xml" \
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC; env VERBOSE=1 make check 2>&1 | tee make_check_gcc.log ; sed -E -e 's/classname\=\"[a-zA-Z0-9_]+/&1_gcc/' unit-test-results.xml > unit-test-results-gcc.xml" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC; env VERBOSE=1 make unit-test-check 2>&1 | tee make_check_gcc.log ; sed -E -e 's/classname\=\"[a-zA-Z0-9_]+/&1_gcc/' unit-test-results.xml > unit-test-results-gcc.xml" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
check_clang: build_clang
docker run --rm --tty \
......@@ -146,7 +163,17 @@ check_clang: build_clang
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG; env VERBOSE=1 make check 2>&1 | tee make_check_clang.log ; sed -E -e 's/classname\=\"[a-zA-Z0-9_]+/&1_clang/' unit-test-results.xml > unit-test-results-clang.xml" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
style_clang: build_clang
docker run --rm --tty \
${VOLUME} \
${DOCKER_RUN_ENV} \
--env GTEST_OUTPUT="xml:${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG/unit-test-results.xml" \
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG; env VERBOSE=1 make style-check 2>&1 | tee make_style_check_clang.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
# Install
......@@ -160,7 +187,7 @@ install_gcc: check_gcc
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-GCC ; test -d ngraph_dist && rm -fr ngraph_dist && echo 'Removed old ngraph_dist directory' ; make install 2>&1 | tee make_install_gcc.log ; tar czf ngraph_dist_gcc.tgz ngraph_dist 2>&1 | tee make_tarball_gcc.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
install_clang: check_clang
# Puts ngraph_dist in BUILD-CLANG directory. This is used by Jenkins ngraph-tensorflow batch job.
......@@ -170,7 +197,7 @@ install_clang: check_clang
--env RUN_UID="$(shell id -u)" \
--env RUN_CMD="set -e ; set -o pipefail; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD-CLANG ; test -d ngraph_dist && rm -fr ngraph_dist && echo 'Removed old ngraph_dist directory' ; make install 2>&1 | tee make_install_clang.log ; tar czf ngraph_dist_clang.tgz ngraph_dist 2>&1 | tee make_tarball_clang.log" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
# Interactive shell
......@@ -181,7 +208,7 @@ shell: build_docker_image
${DOCKER_RUN_ENV} \
--env RUN_UID="$(shell id -u)" \
"build_ngraph_cpp:${DBUILD_VERSION}" \
sh -c "cd ${DOCKUSER_HOME} ; ${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
# Clean
......
#! /bin/bash
# This script is designed to simulate running as a user with a particular UID
# within a docker container.
#
# Normally a docker container runs as root, which can cause problems with file
# ownership when a host directory tree is mounted into the docker container.
# There are other problems with building and running software as root as
# well. Good practice when validating software builds in a docker container
# is to run as a normal user, since many (most?) end users will not be building
# and installing software as root.
#
# This script should be run using "docker run", with RUN_UID (set to the user
# you want to run as) passed into the docker container as an environment
# variable. The script will then add the UID as user "dockuser" to
# /etc/passwd (important for some software, like bazel), add the new dockuser
# to the sudo group (whether or not sudo is installed), and su to a new shell
# as the dockuser (passing in the existing environment, which is important).
#
# If the environment variable RUN_CMD is passed into the docker container, then
# this script will use RUN_CMD as a command to run when su'ing. If RUN_CMD is
# not defined, then /bin/bash will run, which effectively provides an
# interactive shell in the docker container, for debugging.
set -e # Make sure we exit on any command that returns non-zero
set -u # No unset variables
if [ -z "$RUN_UID" ] ; then
# >&2 redirects echo output to stderr.
# See: https://stackoverflow.com/questions/2990414/echo-that-outputs-to-stderr
( >&2 echo 'ERROR: Environment variable RUN_UID was not set when run-as-user.sh was run' )
( >&2 echo ' Running as default user (root, in docker)' )
( >&2 echo ' ' )
exit 1
else
# The username used in the docker container to map the caller UID to
#
# Note 'dockuser' is used in other scripts, notably Makefile. If you
# choose to change it here, then you need to change it in all other
# scripts, or else the builds will break.
#
DOCK_USER='dockuser'
# We will be su'ing using a non-login shell or command, and preserving
# the environment. This is done so that env. variables passed in with
# "docker run --env ..." are honored.
# Therefore, we need to reset at least HOME=/root ...
#
# Note also that /home/dockuser is used in other scripts, notably
# Makefile. If you choose to change it here, then you need to change it
# in all other scripts, or else the builds will break.
#
export HOME="/home/${DOCK_USER}"
# Make sure the home directory is owned by the new user
if [ -d "${HOME}" ] ; then
chown "${RUN_UID}" "${HOME}"
fi
# Add a user with UID of person running docker (in ${RUN_UID})
# If $HOME does not yet exist, then it will be created
adduser -c 'Docker-User' -u "${RUN_UID}" "${DOCK_USER}"
passwd -d "${DOCK_USER}"
# Add dockuser to the sudo group. Sudo *is* used for installing packages,
# so make sure dockuser can run sudo without requesting a password.
usermod -aG wheel "${DOCK_USER}"
echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
if [ -z "${RUN_CMD+x}" ] ; then # Launch a shell as dockuser
echo 'Running interactive shell (/bin/bash) as dockuser'
su -m "${DOCK_USER}" -c "/bin/bash"
else # Run command as dockuser
echo "Running command [${RUN_CMD}] as dockuser"
su -m "${DOCK_USER}" -c "${RUN_CMD}"
fi
fi
#! /bin/bash
# This script is designed to simulate running as a user with a particular UID
# within a docker container.
#
# Normally a docker container runs as root, which can cause problems with file
# ownership when a host directory tree is mounted into the docker container.
# There are other problems with building and running software as root as
# well. Good practice when validating software builds in a docker container
# is to run as a normal user, since many (most?) end users will not be building
# and installing software as root.
#
# This script should be run using "docker run", with RUN_UID (set to the user
# you want to run as) passed into the docker container as an environment
# variable. The script will then add the UID as user "dockuser" to
# /etc/passwd (important for some software, like bazel), add the new dockuser
# to the sudo group (whether or not sudo is installed), and su to a new shell
# as the dockuser (passing in the existing environment, which is important).
#
# If the environment variable RUN_CMD is passed into the docker container, then
# this script will use RUN_CMD as a command to run when su'ing. If RUN_CMD is
# not defined, then /bin/bash will run, which effectively provides an
# interactive shell in the docker container, for debugging.
set -e # Make sure we exit on any command that returns non-zero
set -u # No unset variables
if [ -z "$RUN_UID" ] ; then
# >&2 redirects echo output to stderr.
# See: https://stackoverflow.com/questions/2990414/echo-that-outputs-to-stderr
( >&2 echo 'ERROR: Environment variable RUN_UID was not set when run-as-user.sh was run' )
( >&2 echo ' Running as default user (root, in docker)' )
( >&2 echo ' ' )
exit 1
else
# The username used in the docker container to map the caller UID to
#
# Note 'dockuser' is used in other scripts, notably Makefile. If you
# choose to change it here, then you need to change it in all other
# scripts, or else the builds will break.
#
DOCK_USER='dockuser'
# We will be su'ing using a non-login shell or command, and preserving
# the environment. This is done so that env. variables passed in with
# "docker run --env ..." are honored.
# Therefore, we need to reset at least HOME=/root ...
#
# Note also that /home/dockuser is used in other scripts, notably
# Makefile. If you choose to change it here, then you need to change it
# in all other scripts, or else the builds will break.
#
export HOME="/home/${DOCK_USER}"
# Make sure the home directory is owned by the new user
if [ -d "${HOME}" ] ; then
chown "${RUN_UID}" "${HOME}"
fi
# Add a user with UID of person running docker (in ${RUN_UID})
# If $HOME does not yet exist, then it will be created
adduser --disabled-password --gecos 'Docker-User' -u "${RUN_UID}" "${DOCK_USER}"
# Add dockuser to the sudo group
adduser "${DOCK_USER}" sudo
# If root access is needed in the docker image while running as a normal
# user, uncomment this and add 'sudo' as a package installed in Dockerfile
# echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
if [ -z "${RUN_CMD+x}" ] ; then # Launch a shell as dockuser
su -m "${DOCK_USER}" -c /bin/bash
else # Run command as dockuser
su -m "${DOCK_USER}" -c "${RUN_CMD}"
fi
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment