Makefile 13.2 KB
Newer Older
1
# ******************************************************************************
2
# Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************

17 18 19
# Basic Makefile for contrib/docker. This can be expanded later as more targets
# are added.

20 21 22 23 24 25 26 27
# Building LLVM from source has been observed to trigger the oom-killer
#    on systems with a large number of cores
#    running with make -j
#
# Default is to build with -j 22 for parallel cmake/make.
# Override with make PARALLEL="-j <num_parallel_processes>" where
#    <num_parallel_processes> = the number of make processes to run in parallel
# Turn off with make PARALLEL=
28
PARALLEL=22
29 30 31 32 33

# DIR is an internal variable that serves as an anchor to this cloned git
# repository.  DIR is mounted into the docker container, so that builds
# can occur within the container on this cloned git repository.  DIR should
# not be modified - if it is, then the build system will not work.
34
DIR = $(realpath ../..)
35 36 37 38 39 40 41 42 43

# DOCKUSER_HOME is the location of the home directory of the fabricated
# "dockuser" user, used only within the docker containers.  "dockuser" is
# created (from the passed-in RUN_UID) to map the docker-caller user's UID to a
# first-class user (/etc/passwd entry, member of sudo group, proper home dir)
# /home/dockuser is also used in other scripts, notably run_as_user.sh, so if
# changed it must be done in other areas for the builds to work.
DOCKUSER_HOME=/home/dockuser

44 45 46
# Use /home/dockuser/ngraph-test, because we run as the user (and not root)
# /root/ngraph-test is not used, because /root is not accessible to user
VOLUME = -v "${DIR}:${DOCKUSER_HOME}/ngraph-test"
47
GIT_COMMIT = $(shell git rev-parse HEAD)
48 49
DBUILD_VERSION = ${GIT_COMMIT}_${PYTHON_VERSION}

50 51 52
# Look for evidence if GPU backend is supported on the platform
NVIDIA_SMI = $(shell which nvidia-smi)

53 54
# Enable additional options to be added on the command line
ifndef CMAKE_OPTIONS_EXTRA
55 56 57
    CMAKE_OPTIONS_EXTRA=
endif

58
# Allow linking pre-built third-party cache files (future)
59 60 61 62
ifndef THIRD_PARTY_CACHE_DIR
    THIRD_PARTY_CACHE_DIR=
endif

63 64 65 66 67 68
# OS set to 'ubuntu1604' by default
# can be overridden on the command line with 'make <target> OS=centos74"
ifndef OS
    OS="ubuntu1604"
endif

69 70
DBUILD_DIR = ${DIR}/contrib/docker/.build-${DBUILD_VERSION}_${OS}

71
# Configuration for specific reference OS in Dockerfiles
72
ifeq ("$(shell echo ${OS} | grep centos)","centos74")
73
    RUN_AS_USER_SCRIPT=${DOCKUSER_HOME}/ngraph-test/contrib/docker/run_as_centos_user.sh
74
    CPU_DOCKERFILE=Dockerfile.ngraph.centos74
75
else
76 77
    CPU_DOCKERFILE="Dockerfile.ngraph.ubuntu1604"
    RUN_AS_USER_SCRIPT=${DOCKUSER_HOME}/ngraph-test/contrib/docker/run_as_ubuntu_user.sh
78
    CMAKE_OPTIONS_EXTRA+=-DNGRAPH_USE_PREBUILT_LLVM=TRUE
79 80
endif

81 82 83 84 85 86 87 88 89 90 91 92 93 94
# Build GPU backend if NVIDIA_SMI command is found
# Sets CMAKE_OPTIONS_EXTRA to introduce GPU build configuration to cmake
# Configuration for GPU backend in Dockerfiles with "_gpu" suffix
# The nvidia-docker command must be used for any targets that actually utilize GPU devices
ifneq ("$(shell echo ${NVIDIA_SMI} | grep nvidia-smi)","")
    DOCKERFILE=${CPU_DOCKERFILE}_gpu
    DOCKER_CMD=nvidia-docker
    DOCKER_CMD_MESG=GPU appears to be supported on this platform.  Building for GPU and CPU backend support.
else
    DOCKERFILE=${CPU_DOCKERFILE}
    DOCKER_CMD=docker
    DOCKER_CMD_MESG=GPU does not appear to be supported on this platform.  Building for CPU backend support only.
endif

95 96
# For gcc builds, we do NOT regard warnings as errors
# For clang builds, we DO make warnings into errors
97
CMAKE_OPTIONS_COMMON=-DNGRAPH_BUILD_DOXYGEN_DOCS=ON -DNGRAPH_BUILD_SPHINX_DOCS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo $(CMAKE_OPTIONS_EXTRA)
98 99
CMAKE_OPTIONS_GCC=$(CMAKE_OPTIONS_COMMON) -DCMAKE_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-test/BUILD-GCC/ngraph_dist
CMAKE_OPTIONS_CLANG=$(MAKE_OPTIONS_COMMON)-DCMAKE_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-test/BUILD-CLANG/ngraph_dist -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 -DNGRAPH_WARNINGS_AS_ERRORS=ON -DNGRAPH_USE_PREBUILT_LLVM=TRUE
100 101 102 103

CALLER_UID := $(shell id -u)
CALLER_GID := $(shell id -g)

104
# Default version is python 2, but can be switched to 3 from command
105 106 107
# line
PYTHON_VERSION = 2

108
# Some targets are DEPRECATED and will be removed at a later date: check_cpu build_ngraph_cpu
109 110
# These DEPRECATED targets are currently included for Jenkins job compatibility with older dev branches
# Please see comments for individual targets for more details
111
.PHONY: clean build_docker_image build_gcc check_gcc build_clang check_clang install_gcc install_clang shell check_cpu build_all build_ngraph_cpu
112 113 114 115 116

DOCKER_BUILD=docker build --rm=true

ifdef http_proxy
DOCKER_BUILD+=--build-arg http_proxy=$(http_proxy)
117
DOCKER_RUN_ENV+=--env "http_proxy=$(http_proxy)"
118 119
endif

120
ifdef https_proxy
121
DOCKER_BUILD+=--build-arg https_proxy=$(https_proxy)
122
DOCKER_RUN_ENV+=--env "https_proxy=$(https_proxy)"
123 124
endif

125 126 127 128
all: check_gcc check_clang

# Docker actions

129
# Isolate specific dockerfiles in a .build_* subdirectory
130
expand_dockerfile_templates:
131 132 133
	@echo "OS=${OS}"
	@echo "DOCKERFILE=${DOCKERFILE}"
	@echo "RUN_AS_USER_SCRIPT=${RUN_AS_USER_SCRIPT}"
134
	cd "${DIR}"/contrib/docker
135
	mkdir "${DBUILD_DIR}" || true
136
	sed -e 's/\(FROM ngraph.*\)/\1:${DBUILD_VERSION}/' ${DOCKERFILE} > "${DBUILD_DIR}/Dockerfile.build_ngraph_${OS}"
137

138
build_docker_image: expand_dockerfile_templates
139
	@echo "OS=${OS}"
140
	@echo ${DBUILD_DIR}
141 142
	export CONTEXTDIR=${DBUILD_DIR};export DOCKER_TAG=build_ngraph_${OS};./make-dimage.sh
	docker tag build_ngraph_${OS}:latest build_ngraph_${OS}:${DBUILD_VERSION}
143

144
build_docker: build_docker_image
145

L.S. Cook's avatar
L.S. Cook committed
146
# Build docs
147
docs: sphinx_doc
148

149
# Docs build does not depend on GPU dependencies
150
sphinx_doc: build_docker_image
L.S. Cook's avatar
L.S. Cook committed
151 152 153
	# sphinx html docs build
	docker run --rm --tty \
            ${VOLUME} \
154
	    ${DOCKER_RUN_ENV} \
L.S. Cook's avatar
L.S. Cook committed
155
            --env RUN_UID="$(shell id -u)" \
156
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-docs.sh" \
157
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
158
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
L.S. Cook's avatar
L.S. Cook committed
159 160

# Build
161 162
build_all: build_gcc build_clang

163 164 165
# Build targets ALWAYS clean build directories (BUILD-GCC, BUILD-CLANG) prior to building
# Always use docker command to build docker images
# nvidia-docker command is not appropriate
166
build_gcc: build_docker_image
167 168 169
	@echo ""
	@echo "${DOCKER_CMD_MESG}"
	@echo ""
170 171 172
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
173
	    --env BUILD_SUBDIR=BUILD-GCC \
174
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
175 176 177
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='build_gcc' \
178
            --env RUN_UID="$(shell id -u)" \
179
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
180
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
181
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
182

183 184 185
# Build targets ALWAYS clean build directories (BUILD-GCC, BUILD-CLANG) prior to building
# Always use docker command to build docker images
# nvidia-docker command is not appropriate
186
build_clang: build_docker_image
187 188 189
	@echo ""
	@echo "${DOCKER_CMD_MESG}"
	@echo ""
190
	docker run --rm --tty \
191 192
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
193
	    --env BUILD_SUBDIR=BUILD-CLANG \
194
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
195 196 197
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='build_clang' \
198
            --env RUN_UID="$(shell id -u)" \
199
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
200
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
201
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
202 203 204 205

# Check (run unit-tests)
check_all: check_gcc check_clang

206 207 208 209
# Always use the platform-specific docker command to run unit tests
# ngraph make check target executes unit-test-check and style-check
check_gcc: build_gcc
	${DOCKER_CMD} run --rm --tty \
210 211 212
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
	    --env BUILD_SUBDIR=BUILD-GCC \
213
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
214 215 216 217 218
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=check_gcc \
            --env RUN_UID="$(shell id -u)" \
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
219
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
220 221
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"

222 223 224 225
# Always use the platform-specific docker command to run unit tests
# ngraph make check target executes unit-test-check and style-check
check_clang: build_clang
	${DOCKER_CMD} run --rm --tty \
226 227
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
228
	    --env BUILD_SUBDIR=BUILD-CLANG \
229
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
230 231 232
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=check_clang \
233
            --env RUN_UID="$(shell id -u)" \
234
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
235
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
236
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
237

238
# Always use the platform-specific docker command to run unit tests
239
unit_test_check_gcc: build_gcc
240
	${DOCKER_CMD} run --rm --tty \
241 242
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
243
	    --env BUILD_SUBDIR=BUILD-GCC \
244
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
245 246 247
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='unit-test-check_gcc' \
248
            --env RUN_UID="$(shell id -u)" \
249
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
250
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
251 252
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"

253
# Always use the platform-specific docker command to run unit tests
254
unit_test_check_clang: build_clang
255
	${DOCKER_CMD} run --rm --tty \
256 257
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
258
	    --env BUILD_SUBDIR=BUILD-CLANG \
259
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
260 261 262
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='unit-test-check_clang' \
263
            --env RUN_UID="$(shell id -u)" \
264
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
265
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
266
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
267

268 269
style_check_clang: build_clang

270 271 272 273
# Install

install_all: install_gcc install_clang

274 275 276
# install targets do not depend on GPU dependencies
# no unit tests are executed
# build prerequisites include GPU dependencies in the docker image automatically
277
install_gcc: build_gcc
278 279 280
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
281
	    --env BUILD_SUBDIR=BUILD-GCC \
282
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
283 284 285
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=install_gcc \
286
            --env RUN_UID="$(shell id -u)" \
287
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
288
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
289
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
290

291 292 293
# install targets do not depend on GPU dependencies
# no unit tests are executed
# build prerequisites include GPU dependencies in the docker image automatically
294
install_clang: build_clang
295 296 297
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
298
	    --env BUILD_SUBDIR=BUILD-CLANG \
299
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
300 301 302
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=install_clang \
303
            --env RUN_UID="$(shell id -u)" \
304
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
305
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
306
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
307 308 309

# Interactive shell

310
# Always use the platform-specific docker command for the interactive shell
311 312
shell: build_docker_image
	# "make shell" runs an interactive shell in the docker image, for debugging
313 314
	@echo "${DOCKER_CMD_MESG}"
	${DOCKER_CMD} run --rm --tty --interactive \
315 316 317
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
            --env RUN_UID="$(shell id -u)" \
318
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
319
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
320 321 322 323 324 325 326 327

# Clean

clean:
	rm -f "${DIR}"/contrib/docker/.build-*/Dockerfile.* || echo "keep going if files are not present"
	rmdir "${DIR}"/contrib/docker/.build-* || echo "keep going if directory is not present"
	rm -fr "${DIR}"/BUILD-GCC
	rm -fr "${DIR}"/BUILD-CLANG
328
	rm -fr "${DIR}"/BUILD-DOCS