Ninja build (#4101)

* Add new option to do a clean Ninja build.

* Enable Interpreter and unit-test.

* Enable json.

* Enable NOP backend.

* Enable tools.

* Enable CPU DEX ONLY on Linux.

* Quick hack to enable dex only on Mac.

* Enable onnx import.

* Cleanup gtest.

* Cleanup tbb_prebuilt.

* Fix typo.

* Move gtest include point.

* Fix gtest windows path.

* Remove temporary option.

* Keep GCPU upto date with INTERPRETER.

* Add missing file.

* Enable GCPU.

* Enable CPU codegen.

* Remove tabs.

* Revert "Merge branch 'silee2/gcpu' into silee2/ninja"

This reverts commit e513cd3fe3659a477af6e8fab6c1852a5b1493c3, reversing
changes made to 31dca3301cd89e15af23acd0d50aa06a1c5d6001.

* Enable PlaidML.
Co-authored-by: aslepko <44713115+aslepko@users.noreply.github.com>

CMakeLists.txt

@@ -114,6 +114,7 @@ ngraph_var(NGRAPH_USE_PREBUILT_MLIR DEFAULT "FALSE")
 ngraph_var(NGRAPH_USE_PREBUILT_LLVM DEFAULT "FALSE")
 
 option(NGRAPH_UNIT_TEST_ENABLE "Control the building of unit tests" TRUE)
+option(NGRAPH_TEST_UTIL_ENABLE "Control the building of test utility" TRUE)
 option(NGRAPH_DOC_BUILD_ENABLE "Control the building of documentation" FALSE)
 option(NGRAPH_TOOLS_ENABLE "Control the building of tool" TRUE)
 option(NGRAPH_CPU_ENABLE "Control the building of the CPU backend" TRUE)
@@ -188,6 +189,13 @@ if (NOT NGRAPH_JSON_ENABLE)
     set(NGRAPH_TOOLS_ENABLE FALSE)
 endif()
 
+if (NGRAPH_UNIT_TEST_ENABLE)
+    if (NOT NGRAPH_TEST_UTIL_ENABLE)
+        message(WARNING "Unit test needs test util: Turning NGRAPH_TEST_UTIL_ENABLE ON!")
+        set(NGRAPH_TEST_UTIL_ENABLE TRUE)
+    endif()
+endif()
+
 macro (NORMALIZE_BOOL VAL)
     if (${VAL})
         set(${VAL} ON)
@@ -197,6 +205,7 @@ macro (NORMALIZE_BOOL VAL)
 endmacro()
 
 NORMALIZE_BOOL(NGRAPH_UNIT_TEST_ENABLE)
+NORMALIZE_BOOL(NGRAPH_TEST_UTIL_ENABLE)
 NORMALIZE_BOOL(NGRAPH_DOC_BUILD_ENABLE)
 NORMALIZE_BOOL(NGRAPH_TOOLS_ENABLE)
 NORMALIZE_BOOL(NGRAPH_CPU_ENABLE)
@@ -229,6 +238,7 @@ if (NGRAPH_CPU_ENABLE)
 endif()
 
 message(STATUS "NGRAPH_UNIT_TEST_ENABLE:              ${NGRAPH_UNIT_TEST_ENABLE}")
+message(STATUS "NGRAPH_TEST_UTIL_ENABLE:              ${NGRAPH_TEST_UTIL_ENABLE}")
 message(STATUS "NGRAPH_DOC_BUILD_ENABLE:              ${NGRAPH_DOC_BUILD_ENABLE}")
 message(STATUS "NGRAPH_TOOLS_ENABLE:                  ${NGRAPH_TOOLS_ENABLE}")
 message(STATUS "NGRAPH_CPU_ENABLE:                    ${NGRAPH_CPU_ENABLE}")
@@ -261,10 +271,7 @@ if (NGRAPH_CPU_ENABLE)
     message(STATUS "NGRAPH_TBB_ENABLE:                    ${NGRAPH_TBB_ENABLE}")
 endif()
 
-if("${CMAKE_GENERATOR}" STREQUAL "Ninja" AND (NGRAPH_TOOLS_ENABLE OR NGRAPH_CPU_ENABLE OR NGRAPH_MLIR_ENABLE OR
-        NGRAPH_INTELGPU_ENABLE OR NGRAPH_GPU_ENABLE OR NGRAPH_INTERPRETER_ENABLE OR NGRAPH_NOP_ENABLE OR
-        NGRAPH_GPUH_ENABLE OR NGRAPH_GENERIC_CPU_ENABLE OR NGRAPH_ONNX_IMPORT_ENABLE OR NGRAPH_PLAIDML_ENABLE OR
-        NGRAPH_JSON_ENABLE))
+if("${CMAKE_GENERATOR}" STREQUAL "Ninja" AND NGRAPH_MLIR_ENABLE)
     message(FATAL_ERROR "Ninja generator not supported.")
 endif()
 
@@ -595,7 +602,6 @@ if (NGRAPH_ONNX_IMPORT_ENABLE)
     endif()
 endif()
 
-include(cmake/external_gtest.cmake)
 if(NGRAPH_JSON_ENABLE)
     include(cmake/external_json.cmake)
 endif()
@@ -642,6 +648,10 @@ endif()
 
 add_subdirectory(src)
 
+if (NGRAPH_TEST_UTIL_ENABLE)
+    include(cmake/external_gtest.cmake)
+endif()
+
 add_subdirectory(test)
 add_subdirectory(doc/examples)
 

cmake/external_gtest.cmake

@@ -33,6 +33,7 @@ if(WIN32)
         -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG=${GTEST_OUTPUT_DIR}
         -Dgtest_force_shared_crt=TRUE
     )
+    set(GMOCK_OUTPUT_DIR ${GTEST_OUTPUT_DIR})
 endif()
 
 if(CMAKE_BUILD_TYPE)
@@ -50,15 +51,10 @@ else()
 endif()
 
 #Build for ninja
-if(UNIX)
-    SET(GTEST_PATHS ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/gtest/libgtest.a
-            ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/libgmock.a)
-else()
-    SET(GTEST_PATHS ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/gtest/gtest.lib
-            ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/gtest/gmock.lib
-            ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/gtest/gtestd.lib
-            ${CMAKE_BINARY_DIR}/ngraph/gtest/build/googlemock/gtest/gmockd.lib)
-endif()
+SET(GTEST_PATHS ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
+    ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
+    ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
+    ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
 
 ExternalProject_Add(
     ext_gtest
@@ -90,18 +86,12 @@ target_include_directories(libgtest SYSTEM INTERFACE
     ${SOURCE_DIR}/googletest/include
     ${SOURCE_DIR}/googlemock/include)
 
-if(LINUX OR APPLE)
+if(LINUX OR APPLE OR WIN32)
     target_link_libraries(libgtest INTERFACE
         debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
         debug ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
         optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
         optimized ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
-elseif(WIN32)
-    target_link_libraries(libgtest INTERFACE
-        debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
-        debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
-        optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
-        optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
 else()
     message(FATAL_ERROR "libgtest: Unsupported platform.")
 endif()

cmake/external_mkldnn.cmake

@@ -131,6 +131,8 @@ ExternalProject_Add(
     UPDATE_COMMAND ""
     DOWNLOAD_NO_PROGRESS TRUE
     EXCLUDE_FROM_ALL TRUE
+    BUILD_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLML_LIB}
+                     ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${OMP_LIB}
 )
 
 set(MKLDNN_DEPENDS ext_mkl)
@@ -260,6 +262,7 @@ else()
         BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/mkldnn/build"
         INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/mkldnn"
         EXCLUDE_FROM_ALL TRUE
+        BUILD_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB}
         )
 endif()
 

cmake/external_mkldnn_v1.cmake

@@ -131,6 +131,8 @@ ExternalProject_Add(
     UPDATE_COMMAND ""
     DOWNLOAD_NO_PROGRESS TRUE
     EXCLUDE_FROM_ALL TRUE
+    BUILD_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLML_LIB}
+                     ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${OMP_LIB}
 )
 
 set(MKLDNN_DEPENDS ext_mkl)
@@ -260,6 +262,7 @@ else()
         BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/mkldnn/build"
         INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/mkldnn"
         EXCLUDE_FROM_ALL TRUE
+        BUILD_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB}
         )
 endif()
 

cmake/external_onnx.cmake

@@ -55,6 +55,7 @@ ExternalProject_Add(
     BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/onnx/bin"
     INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/onnx"
     EXCLUDE_FROM_ALL TRUE
+    BUILD_BYPRODUCTS ${EXTERNAL_PROJECTS_ROOT}/onnx/bin/libonnx.a ${EXTERNAL_PROJECTS_ROOT}/onnx/bin/libonnx_proto.a
 )
 
 # -----------------------------------------------------------------------------

cmake/external_protobuf.cmake

@@ -30,6 +30,21 @@ else()
     set(NGRAPH_PROTOBUF_GIT_TAG "v3.6.1")
 endif()
 
+set(Protobuf_INSTALL_PREFIX ${EXTERNAL_PROJECTS_ROOT}/protobuf)
+set(Protobuf_PROTOC_EXECUTABLE ${Protobuf_INSTALL_PREFIX}/bin/protoc)
+set(Protobuf_INCLUDE_DIR ${Protobuf_INSTALL_PREFIX}/include)
+if (WIN32)
+    set(Protobuf_LIBRARY ${Protobuf_INSTALL_PREFIX}/lib/libprotobuf.lib)
+else()
+    set(Protobuf_LIBRARY ${Protobuf_INSTALL_PREFIX}/lib/libprotobuf.a)
+endif()
+
+if ("${CMAKE_GENERATOR}" STREQUAL "Ninja")
+    set(MAKE_UTIL make)
+else()
+    set(MAKE_UTIL $(MAKE))
+endif()
+
 if (WIN32)
     ExternalProject_Add(
         ext_protobuf
@@ -56,6 +71,7 @@ if (WIN32)
         BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/build"
         INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf"
         EXCLUDE_FROM_ALL TRUE
+        BUILD_BYPRODUCTS ${Protobuf_PROTOC_EXECUTABLE} ${Protobuf_LIBRARY}
     )
 elseif (APPLE)
     # Don't manually set compiler on macos since it causes compile error on macos >= 10.14
@@ -75,6 +91,7 @@ elseif (APPLE)
         BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/src"
         INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf"
         EXCLUDE_FROM_ALL TRUE
+        BUILD_BYPRODUCTS ${Protobuf_PROTOC_EXECUTABLE} ${Protobuf_LIBRARY}
         )
 else()
     if (DEFINED NGRAPH_USE_CXX_ABI)
@@ -91,7 +108,7 @@ else()
         UPDATE_COMMAND ""
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${EXTERNAL_PROJECTS_ROOT}/protobuf --disable-shared CXX=${CMAKE_CXX_COMPILER}
-        BUILD_COMMAND $(MAKE) "${BUILD_FLAGS}"
+        BUILD_COMMAND ${MAKE_UTIL} "${BUILD_FLAGS}"
         TMP_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/tmp"
         STAMP_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/stamp"
         DOWNLOAD_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/download"
@@ -99,6 +116,7 @@ else()
         BINARY_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf/src"
         INSTALL_DIR "${EXTERNAL_PROJECTS_ROOT}/protobuf"
         EXCLUDE_FROM_ALL TRUE
+        BUILD_BYPRODUCTS ${Protobuf_PROTOC_EXECUTABLE} ${Protobuf_LIBRARY}
         )
 endif()
 
@@ -106,15 +124,6 @@ endif()
 # Use the interface of FindProtobuf.cmake
 # -----------------------------------------------------------------------------
 
-set(Protobuf_INSTALL_PREFIX ${EXTERNAL_PROJECTS_ROOT}/protobuf)
-set(Protobuf_PROTOC_EXECUTABLE ${Protobuf_INSTALL_PREFIX}/bin/protoc)
-set(Protobuf_INCLUDE_DIR ${Protobuf_INSTALL_PREFIX}/include)
-if (WIN32)
-    set(Protobuf_LIBRARY ${Protobuf_INSTALL_PREFIX}/lib/libprotobuf.lib)
-else()
-    set(Protobuf_LIBRARY ${Protobuf_INSTALL_PREFIX}/lib/libprotobuf.a)
-endif()
-
 if(NGRAPH_ONNX_IMPORT_ENABLE)
     if (NOT TARGET libprotobuf)
         add_library(libprotobuf INTERFACE)

cmake/external_tbb_prebuilt.cmake

@@ -28,6 +28,19 @@ elseif(APPLE)
     set(TBB_SHA1_HASH 36926fb46add578b88a5c7e19652b94bb612e4be)
 endif()
 
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(TBB_LIB_NAME tbb_debug)
+else()
+    set(TBB_LIB_NAME tbb)
+endif()
+
+if (WIN32)
+    set(TBB_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+elseif(APPLE)
+    set(TBB_BYPRODUCTS ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
+
 ExternalProject_Add(
     ext_tbb
     PREFIX tbb
@@ -39,17 +52,12 @@ ExternalProject_Add(
     UPDATE_COMMAND ""
     DOWNLOAD_NO_PROGRESS TRUE
     EXCLUDE_FROM_ALL TRUE
+    BUILD_BYPRODUCTS ${TBB_BYPRODUCTS}
     )
 
 ExternalProject_Get_Property(ext_tbb SOURCE_DIR)
 set(INSTALL_DIR ${SOURCE_DIR}/${NGRAPH_TBB_SUB_VERSION})
 
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(TBB_LIB_NAME tbb_debug)
-else()
-    set(TBB_LIB_NAME tbb)
-endif()
-
 if (WIN32)
     set(TBB_LINK_LIBS ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
 

src/ngraph/frontend/fluid/operators/test/CMakeLists.txt

@@ -24,6 +24,11 @@ if(NOT NGRAPH_CPU_ENABLE)
     return()
 endif()
 
+if(NOT NGRAPH_TEST_UTIL_ENABLE)
+    message(WARNING "Fluid test: Turning on test util!")
+    set(NGRAPH_TEST_UTIL_ENABLE ON)
+endif()
+
 message(STATUS "fluid tests enabled")
 
 if(LINUX)

test/util/CMakeLists.txt

@@ -14,6 +14,10 @@
 # limitations under the License.
 # ******************************************************************************
 
+if(NOT NGRAPH_TEST_UTIL_ENABLE)
+    return()
+endif()
+
 set (SRC
     autodiff/backprop_function.cpp
     all_close_f.cpp