Makefile 13.2 KB
Newer Older
1
# ******************************************************************************
2
# Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************

17 18 19
# Basic Makefile for contrib/docker. This can be expanded later as more targets
# are added.

20 21 22 23 24 25 26 27
# Building LLVM from source has been observed to trigger the oom-killer
#    on systems with a large number of cores
#    running with make -j
#
# Default is to build with -j 22 for parallel cmake/make.
# Override with make PARALLEL="-j <num_parallel_processes>" where
#    <num_parallel_processes> = the number of make processes to run in parallel
# Turn off with make PARALLEL=
28
PARALLEL=22
29 30 31 32 33

# DIR is an internal variable that serves as an anchor to this cloned git
# repository.  DIR is mounted into the docker container, so that builds
# can occur within the container on this cloned git repository.  DIR should
# not be modified - if it is, then the build system will not work.
34
DIR = $(realpath ../..)
35 36 37 38 39 40 41 42 43

# DOCKUSER_HOME is the location of the home directory of the fabricated
# "dockuser" user, used only within the docker containers.  "dockuser" is
# created (from the passed-in RUN_UID) to map the docker-caller user's UID to a
# first-class user (/etc/passwd entry, member of sudo group, proper home dir)
# /home/dockuser is also used in other scripts, notably run_as_user.sh, so if
# changed it must be done in other areas for the builds to work.
DOCKUSER_HOME=/home/dockuser

44 45 46
# Use /home/dockuser/ngraph-test, because we run as the user (and not root)
# /root/ngraph-test is not used, because /root is not accessible to user
VOLUME = -v "${DIR}:${DOCKUSER_HOME}/ngraph-test"
47
GIT_COMMIT = $(shell git rev-parse HEAD)
48 49
DBUILD_VERSION = ${GIT_COMMIT}_${PYTHON_VERSION}

50 51 52
# Look for evidence if GPU backend is supported on the platform
NVIDIA_SMI = $(shell which nvidia-smi)

53 54
# Enable additional options to be added on the command line
ifndef CMAKE_OPTIONS_EXTRA
55 56 57
    CMAKE_OPTIONS_EXTRA=
endif

58
# Allow linking pre-built third-party cache files (future)
59 60 61 62
ifndef THIRD_PARTY_CACHE_DIR
    THIRD_PARTY_CACHE_DIR=
endif

63 64 65 66 67 68
# OS set to 'ubuntu1604' by default
# can be overridden on the command line with 'make <target> OS=centos74"
ifndef OS
    OS="ubuntu1604"
endif

69 70
DBUILD_DIR = ${DIR}/contrib/docker/.build-${DBUILD_VERSION}_${OS}

71
# Configuration for specific reference OS in Dockerfiles
72
ifeq ("$(shell echo ${OS} | grep centos)","centos74")
73
    RUN_AS_USER_SCRIPT=${DOCKUSER_HOME}/ngraph-test/contrib/docker/run_as_centos_user.sh
74
    CPU_DOCKERFILE=Dockerfile.ngraph.centos74
75
else
76 77
    CPU_DOCKERFILE="Dockerfile.ngraph.ubuntu1604"
    RUN_AS_USER_SCRIPT=${DOCKUSER_HOME}/ngraph-test/contrib/docker/run_as_ubuntu_user.sh
78
    CMAKE_OPTIONS_EXTRA+=-DNGRAPH_USE_PREBUILT_LLVM=TRUE
79 80
endif

81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
# Build GPU backend if NVIDIA_SMI command is found
# Sets CMAKE_OPTIONS_EXTRA to introduce GPU build configuration to cmake
# Configuration for GPU backend in Dockerfiles with "_gpu" suffix
# The nvidia-docker command must be used for any targets that actually utilize GPU devices
ifneq ("$(shell echo ${NVIDIA_SMI} | grep nvidia-smi)","")
    CMAKE_OPTIONS_EXTRA+=-DNGRAPH_GPU_ENABLE=TRUE
    DOCKERFILE=${CPU_DOCKERFILE}_gpu
    DOCKER_CMD=nvidia-docker
    DOCKER_CMD_MESG=GPU appears to be supported on this platform.  Building for GPU and CPU backend support.
else
    DOCKERFILE=${CPU_DOCKERFILE}
    DOCKER_CMD=docker
    DOCKER_CMD_MESG=GPU does not appear to be supported on this platform.  Building for CPU backend support only.
endif

96 97
# For gcc builds, we do NOT regard warnings as errors
# For clang builds, we DO make warnings into errors
98
CMAKE_OPTIONS_COMMON=-DNGRAPH_BUILD_DOXYGEN_DOCS=ON -DNGRAPH_BUILD_SPHINX_DOCS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo $(CMAKE_OPTIONS_EXTRA)
99 100
CMAKE_OPTIONS_GCC=$(CMAKE_OPTIONS_COMMON) -DCMAKE_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-test/BUILD-GCC/ngraph_dist
CMAKE_OPTIONS_CLANG=$(MAKE_OPTIONS_COMMON)-DCMAKE_INSTALL_PREFIX=${DOCKUSER_HOME}/ngraph-test/BUILD-CLANG/ngraph_dist -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 -DNGRAPH_WARNINGS_AS_ERRORS=ON -DNGRAPH_USE_PREBUILT_LLVM=TRUE
101 102 103 104

CALLER_UID := $(shell id -u)
CALLER_GID := $(shell id -g)

105
# Default version is python 2, but can be switched to 3 from command
106 107 108
# line
PYTHON_VERSION = 2

109
# Some targets are DEPRECATED and will be removed at a later date: check_cpu build_ngraph_cpu
110 111
# These DEPRECATED targets are currently included for Jenkins job compatibility with older dev branches
# Please see comments for individual targets for more details
112
.PHONY: clean build_docker_image build_gcc check_gcc build_clang check_clang install_gcc install_clang shell check_cpu build_all build_ngraph_cpu
113 114 115 116 117

DOCKER_BUILD=docker build --rm=true

ifdef http_proxy
DOCKER_BUILD+=--build-arg http_proxy=$(http_proxy)
118
DOCKER_RUN_ENV+=--env "http_proxy=$(http_proxy)"
119 120
endif

121
ifdef https_proxy
122
DOCKER_BUILD+=--build-arg https_proxy=$(https_proxy)
123
DOCKER_RUN_ENV+=--env "https_proxy=$(https_proxy)"
124 125
endif

126 127 128 129
all: check_gcc check_clang

# Docker actions

130
# Isolate specific dockerfiles in a .build_* subdirectory
131
expand_dockerfile_templates:
132 133 134
	@echo "OS=${OS}"
	@echo "DOCKERFILE=${DOCKERFILE}"
	@echo "RUN_AS_USER_SCRIPT=${RUN_AS_USER_SCRIPT}"
135
	cd "${DIR}"/contrib/docker
136
	mkdir "${DBUILD_DIR}" || true
137
	sed -e 's/\(FROM ngraph.*\)/\1:${DBUILD_VERSION}/' ${DOCKERFILE} > "${DBUILD_DIR}/Dockerfile.build_ngraph_${OS}"
138

139
build_docker_image: expand_dockerfile_templates
140
	@echo "OS=${OS}"
141
	@echo ${DBUILD_DIR}
142 143
	export CONTEXTDIR=${DBUILD_DIR};export DOCKER_TAG=build_ngraph_${OS};./make-dimage.sh
	docker tag build_ngraph_${OS}:latest build_ngraph_${OS}:${DBUILD_VERSION}
144

145
build_docker: build_docker_image
146

L.S. Cook's avatar
L.S. Cook committed
147
# Build docs
148
docs: sphinx_doc
149

150
# Docs build does not depend on GPU dependencies
151
sphinx_doc: build_docker_image
L.S. Cook's avatar
L.S. Cook committed
152 153 154
	# sphinx html docs build
	docker run --rm --tty \
            ${VOLUME} \
155
	    ${DOCKER_RUN_ENV} \
L.S. Cook's avatar
L.S. Cook committed
156
            --env RUN_UID="$(shell id -u)" \
157
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-docs.sh" \
158
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
159
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
L.S. Cook's avatar
L.S. Cook committed
160 161

# Build
162 163
build_all: build_gcc build_clang

164 165 166
# Build targets ALWAYS clean build directories (BUILD-GCC, BUILD-CLANG) prior to building
# Always use docker command to build docker images
# nvidia-docker command is not appropriate
167
build_gcc: build_docker_image
168 169 170
	@echo ""
	@echo "${DOCKER_CMD_MESG}"
	@echo ""
171 172 173
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
174
	    --env BUILD_SUBDIR=BUILD-GCC \
175
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
176 177 178
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='build_gcc' \
179
            --env RUN_UID="$(shell id -u)" \
180
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
181
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
182
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
183

184 185 186
# Build targets ALWAYS clean build directories (BUILD-GCC, BUILD-CLANG) prior to building
# Always use docker command to build docker images
# nvidia-docker command is not appropriate
187
build_clang: build_docker_image
188 189 190
	@echo ""
	@echo "${DOCKER_CMD_MESG}"
	@echo ""
191
	docker run --rm --tty \
192 193
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
194
	    --env BUILD_SUBDIR=BUILD-CLANG \
195
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
196 197 198
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='build_clang' \
199
            --env RUN_UID="$(shell id -u)" \
200
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
201
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
202
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
203 204 205 206

# Check (run unit-tests)
check_all: check_gcc check_clang

207 208 209 210
# Always use the platform-specific docker command to run unit tests
# ngraph make check target executes unit-test-check and style-check
check_gcc: build_gcc
	${DOCKER_CMD} run --rm --tty \
211 212 213
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
	    --env BUILD_SUBDIR=BUILD-GCC \
214
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
215 216 217 218 219
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=check_gcc \
            --env RUN_UID="$(shell id -u)" \
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
220
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
221 222
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"

223 224 225 226
# Always use the platform-specific docker command to run unit tests
# ngraph make check target executes unit-test-check and style-check
check_clang: build_clang
	${DOCKER_CMD} run --rm --tty \
227 228
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
229
	    --env BUILD_SUBDIR=BUILD-CLANG \
230
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
231 232 233
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=check_clang \
234
            --env RUN_UID="$(shell id -u)" \
235
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
236
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
237
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
238

239
# Always use the platform-specific docker command to run unit tests
240
unit_test_check_gcc: build_gcc
241
	${DOCKER_CMD} run --rm --tty \
242 243
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
244
	    --env BUILD_SUBDIR=BUILD-GCC \
245
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
246 247 248
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='unit-test-check_gcc' \
249
            --env RUN_UID="$(shell id -u)" \
250
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
251
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
252 253
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"

254
# Always use the platform-specific docker command to run unit tests
255
unit_test_check_clang: build_clang
256
	${DOCKER_CMD} run --rm --tty \
257 258
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
259
	    --env BUILD_SUBDIR=BUILD-CLANG \
260
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
261 262 263
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN='unit-test-check_clang' \
264
            --env RUN_UID="$(shell id -u)" \
265
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
266
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
267
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
268

269 270
style_check_clang: build_clang

271 272 273 274
# Install

install_all: install_gcc install_clang

275 276 277
# install targets do not depend on GPU dependencies
# no unit tests are executed
# build prerequisites include GPU dependencies in the docker image automatically
278
install_gcc: build_gcc
279 280 281
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
282
	    --env BUILD_SUBDIR=BUILD-GCC \
283
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
284 285 286
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=install_gcc \
287
            --env RUN_UID="$(shell id -u)" \
288
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
289
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
290
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
291

292 293 294
# install targets do not depend on GPU dependencies
# no unit tests are executed
# build prerequisites include GPU dependencies in the docker image automatically
295
install_clang: build_clang
296 297 298
	docker run --rm --tty \
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
299
	    --env BUILD_SUBDIR=BUILD-CLANG \
300
	    --env CMAKE_OPTIONS_EXTRA="${CMAKE_OPTIONS_EXTRA}" \
301 302 303
	    --env PARALLEL=${PARALLEL} \
	    --env THIRD_PARTY_CACHE_DIR=${THIRD_PARTY_CACHE_DIR} \
	    --env CMD_TO_RUN=install_clang \
304
            --env RUN_UID="$(shell id -u)" \
305
            --env RUN_CMD="${DOCKUSER_HOME}/ngraph-test/contrib/docker/build-ngraph-and-test.sh" \
306
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
307
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
308 309 310

# Interactive shell

311
# Always use the platform-specific docker command for the interactive shell
312 313
shell: build_docker_image
	# "make shell" runs an interactive shell in the docker image, for debugging
314 315
	@echo "${DOCKER_CMD_MESG}"
	${DOCKER_CMD} run --rm --tty --interactive \
316 317 318
            ${VOLUME} \
	    ${DOCKER_RUN_ENV} \
            --env RUN_UID="$(shell id -u)" \
319
            "build_ngraph_${OS}:${DBUILD_VERSION}" \
320
	    sh -c "cd ${DOCKUSER_HOME}; ${RUN_AS_USER_SCRIPT}"
321 322 323 324 325 326 327 328

# Clean

clean:
	rm -f "${DIR}"/contrib/docker/.build-*/Dockerfile.* || echo "keep going if files are not present"
	rmdir "${DIR}"/contrib/docker/.build-* || echo "keep going if directory is not present"
	rm -fr "${DIR}"/BUILD-GCC
	rm -fr "${DIR}"/BUILD-CLANG
329
	rm -fr "${DIR}"/BUILD-DOCS