Cyphers/r15catchup (#2567)

* Fix bug introduced by #2238 on r0.15 (#2515)

* Fix bug introduced by #2238

* style

* Avijit/add diag 4 distributed debugging (#2523)

* Fix bug introduced by #2238

* Added a debug logging macro: NGRAPH_DIST_DEBUG which prints the timestamp and MPI rank for distributed AllReduceops

* Fixed code formatting.

* Moved the logging implementation to log.cpp

* Fixed clang warning

* Don't use namespace in header

* Fix ifdef

* disable warning

* Fix files that had missing headers (#2539) (#2541)

* Update mkl-dnn to v0.18. (#2543)

* Update mkl-dnn to v0.18.

* Update mkl-dnn patch file.

* Fixed Centos build issues (gcc 4.8.5 doesn't implement std::put_time - hence the issue). Also fixed a logical error in the distributed setup for unit tests (#2545)

* Migrate Klocwork issues (#2557)

* Migrate Klocwork fixes

* change to kick off another build

* Fix crash by correcting index (#2561)

* Update build script to fix Windows build issues and add some work tow… (#2531)

* Update build script to fix Windows build issues and add some work towards making codegen work on Windows.

* Use "git apply" instead of "patch". (#2532)

Enable using pre-installed MKLDNN on Window.

* Remove commented out lines.

cmake/external_gtest.cmake

@@ -76,8 +76,18 @@ target_include_directories(libgtest SYSTEM INTERFACE
     ${SOURCE_DIR}/googletest/include
     ${SOURCE_DIR}/googlemock/include)
 
-target_link_libraries(libgtest INTERFACE
-    debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
-    debug ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
-    optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
-    optimized ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
+if(LINUX OR APPLE)
+    target_link_libraries(libgtest INTERFACE
+        debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
+        debug ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
+        optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
+        optimized ${GMOCK_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
+elseif(WIN32)
+    target_link_libraries(libgtest INTERFACE
+        debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtestd${CMAKE_STATIC_LIBRARY_SUFFIX}
+        debug ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmockd${CMAKE_STATIC_LIBRARY_SUFFIX}
+        optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}
+        optimized ${GTEST_OUTPUT_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
+else()
+    message(FATAL_ERROR "libgtest: Unsupported platform.")
+endif()

cmake/external_llvm.cmake

@@ -16,6 +16,8 @@
 
 include(ExternalProject)
 
+set(NGRAPH_LLVM_VERSION 5.0.2)
+
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     if (DEFINED NGRAPH_USE_CXX_ABI)
         set(COMPILE_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=${NGRAPH_USE_CXX_ABI}")
@@ -25,8 +27,8 @@ endif()
 ExternalProject_Add(
     ext_clang
     PREFIX clang
-    GIT_REPOSITORY https://github.com/llvm-mirror/clang.git
-    GIT_TAG 26cac19a0d622afc91cd52a002921074bccc6a27
+    URL http://releases.llvm.org/${NGRAPH_LLVM_VERSION}/cfe-${NGRAPH_LLVM_VERSION}.src.tar.xz
+    URL_HASH SHA1=6581765ec52f8a6354ab56a8e55a8cac1aa5e388
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND ""
@@ -40,8 +42,8 @@ set(CLANG_SOURCE_DIR ${SOURCE_DIR})
 ExternalProject_Add(
     ext_openmp
     PREFIX openmp
-    GIT_REPOSITORY https://github.com/llvm-mirror/openmp.git
-    GIT_TAG 29b515e1e6d26b5b0d32d47d28dcdb4b8a11470d
+    URL http://releases.llvm.org/${NGRAPH_LLVM_VERSION}/openmp-${NGRAPH_LLVM_VERSION}.src.tar.xz
+    URL_HASH SHA1=0e78a7646b63e074e31b6a65e15446af0bdf3c07
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND ""
@@ -58,12 +60,19 @@ else()
     set(LLVM_CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})
 endif()
 
+if(WIN32)
+    set(LLVM_DEPENDS ext_clang)
+else()
+    set(LLVM_DEPENDS ext_clang ext_openmp)
+    set(PASS_LLVM_OPENMP_DIR -DLLVM_EXTERNAL_OPENMP_SOURCE_DIR=${OPENMP_SOURCE_DIR})
+endif()
+
 ExternalProject_Add(
     ext_llvm
     PREFIX llvm
     DEPENDS ext_clang ext_openmp
-    GIT_REPOSITORY https://github.com/llvm-mirror/llvm.git
-    GIT_TAG da4a2839d80ac52958be0129b871beedfe90136e
+    URL http://releases.llvm.org/${NGRAPH_LLVM_VERSION}/llvm-${NGRAPH_LLVM_VERSION}.src.tar.xz
+    URL_HASH SHA1=576d005305335049b89608d897d7ec184d99c6e1
     CMAKE_GENERATOR ${CMAKE_GENERATOR}
     CMAKE_GENERATOR_PLATFORM ${CMAKE_GENERATOR_PLATFORM}
     CMAKE_GENERATOR_TOOLSET ${CMAKE_GENERATOR_TOOLSET}
@@ -71,13 +80,25 @@ ExternalProject_Add(
                 -DCMAKE_ASM_COMPILER=${LLVM_CMAKE_ASM_COMPILER}
                 -DCMAKE_CXX_FLAGS=${COMPILE_FLAGS}
                 -DCMAKE_INSTALL_PREFIX=${EXTERNAL_PROJECTS_ROOT}/llvm
-                -DLLVM_ENABLE_ASSERTIONS=OFF
+                -DLLVM_INCLUDE_DOCS=OFF
                 -DLLVM_INCLUDE_TESTS=OFF
+                -DLLVM_INCLUDE_GO_TESTS=OFF
                 -DLLVM_INCLUDE_EXAMPLES=OFF
-                -DLLVM_BUILD_TOOLS=ON
+                -DLLVM_INCLUDE_BENCHMARKS=OFF
+                -DLLVM_BUILD_TOOLS=OFF
+                -DLLVM_BUILD_UTILS=OFF
+                -DLLVM_BUILD_RUNTIMES=OFF
+                -DLLVM_BUILD_RUNTIME=OFF
                 -DLLVM_TARGETS_TO_BUILD=X86
+                -DLLVM_ENABLE_BINDINGS=OFF
+                -DLLVM_ENABLE_TERMINFO=OFF
+                -DLLVM_ENABLE_ZLIB=OFF
+                -DLIBOMP_OMPT_SUPPORT=OFF
+                -DCLANG_BUILD_TOOLS=OFF
+                -DCLANG_ENABLE_ARCMT=OFF
+                -DCLANG_ENABLE_STATIC_ANALYZER=OFF
                 -DLLVM_EXTERNAL_CLANG_SOURCE_DIR=${CLANG_SOURCE_DIR}
-                -DLLVM_EXTERNAL_OPENMP_SOURCE_DIR=${OPENMP_SOURCE_DIR}
+                ${PASS_LLVM_OPENMP_DIR}
     UPDATE_COMMAND ""
     EXCLUDE_FROM_ALL TRUE
 )
@@ -93,11 +114,7 @@ set(LLVM_LINK_LIBS
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangCodeGen${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangParse${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangSema${CMAKE_STATIC_LIBRARY_SUFFIX}
-    ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangStaticAnalyzerFrontend${CMAKE_STATIC_LIBRARY_SUFFIX}
-    ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangStaticAnalyzerCheckers${CMAKE_STATIC_LIBRARY_SUFFIX}
-    ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangStaticAnalyzerCore${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangAnalysis${CMAKE_STATIC_LIBRARY_SUFFIX}
-    ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangARCMigrate${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangRewriteFrontend${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangEdit${CMAKE_STATIC_LIBRARY_SUFFIX}
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clangAST${CMAKE_STATIC_LIBRARY_SUFFIX}
@@ -160,10 +177,8 @@ set(LLVM_LINK_LIBS
     ${INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}LLVMDemangle${CMAKE_STATIC_LIBRARY_SUFFIX}
 )
 
-if(APPLE)
-    set(LLVM_LINK_LIBS ${LLVM_LINK_LIBS} curses z m)
-else()
-    set(LLVM_LINK_LIBS ${LLVM_LINK_LIBS} tinfo z m)
+if(LINUX OR APPLE)
+    set(LLVM_LINK_LIBS ${LLVM_LINK_LIBS} m)
 endif()
 
 add_library(libllvm INTERFACE)

cmake/external_mkldnn.cmake

@@ -16,10 +16,10 @@
 
 include(ExternalProject)
 
-# Includes blas 3.8.0 in mkldnn 
-set(NGRAPH_MKLDNN_VERSION "v0.18-rc")
-set(NGRAPH_MKLDNN_SUB_VERSION "2019.0.3.20190125")
-set(NGRAPH_MKLDNN_GIT_TAG "08bd90c")
+# Includes blas 3.8.0 in mkldnn
+set(NGRAPH_MKLDNN_VERSION "v0.18")
+set(NGRAPH_MKLDNN_SUB_VERSION "2019.0.3.20190220")
+set(NGRAPH_MKLDNN_GIT_TAG "v0.18")
 
 #------------------------------------------------------------------------------
 # Fetch and install MKL-DNN
@@ -41,15 +41,20 @@ elseif (WIN32)
 endif()
 
 if(MKLDNN_INCLUDE_DIR AND MKLDNN_LIB_DIR)
-    if(NOT LINUX)
+    if(NOT LINUX AND NOT WIN32)
         message(FATAL_ERROR "Unsupported platform for prebuilt mkl-dnn!")
     endif()
     if(NOT MKLML_LIB_DIR)
         set(MKLML_LIB_DIR ${MKLDNN_LIB_DIR})
     endif()
 
-    add_library(libmkldnn SHARED IMPORTED)
-    set_property(TARGET libmkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_LIB})
+    if(WIN32)
+        add_library(libmkldnn STATIC IMPORTED)
+        set_property(TARGET libmkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB})
+    else()
+        add_library(libmkldnn SHARED IMPORTED)
+        set_property(TARGET libmkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_LIB})
+    endif()
     target_include_directories(libmkldnn SYSTEM INTERFACE ${MKLDNN_INCLUDE_DIR})
 
     install(FILES ${MKLDNN_LIB_DIR}/${MKLDNN_LIB} ${MKLML_LIB_DIR}/${MKLML_LIB} ${MKLML_LIB_DIR}/${OMP_LIB}  DESTINATION ${NGRAPH_INSTALL_LIB})
@@ -62,13 +67,13 @@ set(MKLURLROOT "https://github.com/intel/mkl-dnn/releases/download/${NGRAPH_MKLD
 set(MKLVERSION ${NGRAPH_MKLDNN_SUB_VERSION})
 if (LINUX)
     set(MKLPACKAGE "mklml_lnx_${MKLVERSION}.tgz")
-    set(MKL_SHA1_HASH 968318286897da5ffd225f0851aec18f02b347f8)
+    set(MKL_SHA1_HASH b536cd3929ab9ff26a9adc903c92d006d142107b)
 elseif (APPLE)
     set(MKLPACKAGE "mklml_mac_${MKLVERSION}.tgz")
-    set(MKL_SHA1_HASH 8ef2f39b65f23d322af7400d261c3ec883b087c6)
+    set(MKL_SHA1_HASH cb7ccbf73275d5bee19b4d5691c834bf894d1761)
 elseif (WIN32)
     set(MKLPACKAGE "mklml_win_${MKLVERSION}.zip")
-    set(MKL_SHA1_HASH 8383d11b47960e3cd826e2af4b2a7daa9fbd8b68)
+    set(MKL_SHA1_HASH 2f8ae6ef07452c3229bfb7c2aafbae71933d04e3)
 endif()
 set(MKL_LIBS ${MKLML_LIB} ${OMP_LIB})
 set(MKLURL ${MKLURLROOT}${MKLPACKAGE})
@@ -105,7 +110,7 @@ set(MKL_SOURCE_DIR ${source_dir})
 ExternalProject_Add_Step(
     ext_mkl
     CopyMKL
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${MKLML_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${MKLML_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLML_LIB}
     COMMENT "Copy mklml runtime libraries to ngraph build directory."
     DEPENDEES download
     )
@@ -113,7 +118,7 @@ ExternalProject_Add_Step(
 ExternalProject_Add_Step(
     ext_mkl
     CopyOMP
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${OMP_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${OMP_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${OMP_LIB}
     COMMENT "Copy OpenMP runtime libraries to ngraph build directory."
     DEPENDEES download
     )
@@ -122,7 +127,7 @@ if(WIN32)
     ExternalProject_Add_Step(
         ext_mkl
         CopyMKLIMP
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${MKLML_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${MKLML_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${MKLML_IMPLIB}
         COMMENT "Copy mklml runtime libraries to ngraph build directory."
         DEPENDEES download
         )
@@ -130,7 +135,7 @@ if(WIN32)
     ExternalProject_Add_Step(
         ext_mkl
         CopyOMPIMP
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${OMP_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MKL_SOURCE_DIR}/lib/${OMP_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${OMP_IMPLIB}
         COMMENT "Copy OpenMP runtime libraries to ngraph build directory."
         DEPENDEES download
         )
@@ -153,6 +158,9 @@ set(MKLDNN_GIT_TAG ${NGRAPH_MKLDNN_GIT_TAG})
 set(MKLDNN_PATCH_FILE mkldnn.patch)
 set(MKLDNN_LIBS ${EXTERNAL_PROJECTS_ROOT}/mkldnn/lib/${MKLDNN_LIB})
 
+# Revert prior changes to make incremental build work.
+set(MKLDNN_PATCH_REVERT_COMMAND cd ${EXTERNAL_PROJECTS_ROOT}/mkldnn/src && git reset HEAD --hard)
+
 if (WIN32)
     ExternalProject_Add(
         ext_mkldnn
@@ -162,14 +170,8 @@ if (WIN32)
         GIT_TAG ${MKLDNN_GIT_TAG}
         UPDATE_COMMAND ""
         CONFIGURE_COMMAND
-        # Patch gets mad if it applied for a second time so:
-        #    --forward tells patch to ignore if it has already been applied
-        #    --reject-file tells patch to not right a reject file
-        #    || exit 0 changes the exit code for the PATCH_COMMAND to zero so it is not an error
-        # I don't like it, but it works
-        PATCH_COMMAND patch -p1 --forward --reject-file=- -i ${CMAKE_SOURCE_DIR}/cmake/${MKLDNN_PATCH_FILE} || exit 0
-        # Uncomment below with any in-flight MKL-DNN patches
-        # PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/third-party/patches/mkldnn-cmake-openmp.patch
+        PATCH_COMMAND ${MKLDNN_PATCH_REVERT_COMMAND}
+        COMMAND git apply --ignore-space-change --ignore-whitespace ${CMAKE_SOURCE_DIR}/cmake/${MKLDNN_PATCH_FILE}
         CMAKE_GENERATOR ${CMAKE_GENERATOR}
         CMAKE_GENERATOR_PLATFORM ${CMAKE_GENERATOR_PLATFORM}
         CMAKE_GENERATOR_TOOLSET ${CMAKE_GENERATOR_TOOLSET}
@@ -198,14 +200,8 @@ else()
         GIT_TAG ${MKLDNN_GIT_TAG}
         UPDATE_COMMAND ""
         CONFIGURE_COMMAND
-        # Patch gets mad if it applied for a second time so:
-        #    --forward tells patch to ignore if it has already been applied
-        #    --reject-file tells patch to not right a reject file
-        #    || exit 0 changes the exit code for the PATCH_COMMAND to zero so it is not an error
-        # I don't like it, but it works
-        PATCH_COMMAND patch -p1 --forward --reject-file=- -i ${CMAKE_SOURCE_DIR}/cmake/${MKLDNN_PATCH_FILE} || exit 0
-        # Uncomment below with any in-flight MKL-DNN patches
-        # PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/third-party/patches/mkldnn-cmake-openmp.patch
+        PATCH_COMMAND ${MKLDNN_PATCH_REVERT_COMMAND}
+        COMMAND git apply ${CMAKE_SOURCE_DIR}/cmake/${MKLDNN_PATCH_FILE}
         CMAKE_GENERATOR ${CMAKE_GENERATOR}
         CMAKE_GENERATOR_PLATFORM ${CMAKE_GENERATOR_PLATFORM}
         CMAKE_GENERATOR_TOOLSET ${CMAKE_GENERATOR_TOOLSET}
@@ -228,19 +224,27 @@ else()
         )
 endif()
 
-ExternalProject_Add_Step(
-    ext_mkldnn
-    CopyMKLDNN
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${EXTERNAL_PROJECTS_ROOT}/mkldnn/${CMAKE_INSTALL_LIBDIR}/${MKLDNN_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}
-    COMMENT "Copy mkldnn runtime libraries to ngraph build directory."
-    DEPENDEES install
-    )
-
 if(WIN32)
+    ExternalProject_Add_Step(
+        ext_mkldnn
+        CopyMKLDNN
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${EXTERNAL_PROJECTS_ROOT}/mkldnn/bin/${MKLDNN_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB}
+        COMMENT "Copy mkldnn runtime libraries to ngraph build directory."
+        DEPENDEES install
+        )
+
     ExternalProject_Add_Step(
         ext_mkldnn
         CopyMKLDNNIMP
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${EXTERNAL_PROJECTS_ROOT}/mkldnn/${CMAKE_INSTALL_LIBDIR}/${MKLDNN_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${EXTERNAL_PROJECTS_ROOT}/mkldnn/lib/${MKLDNN_IMPLIB} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${MKLDNN_IMPLIB}
+        COMMENT "Copy mkldnn runtime libraries to ngraph build directory."
+        DEPENDEES install
+        )
+else()
+    ExternalProject_Add_Step(
+        ext_mkldnn
+        CopyMKLDNN
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${EXTERNAL_PROJECTS_ROOT}/mkldnn/${CMAKE_INSTALL_LIBDIR}/${MKLDNN_LIB} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB}
         COMMENT "Copy mkldnn runtime libraries to ngraph build directory."
         DEPENDEES install
         )

cmake/external_tbb_prebuilt.cmake

@@ -56,7 +56,7 @@ if (WIN32)
     ExternalProject_Add_Step(
         ext_tbb
         CopyTBB
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/bin/intel64/vc14/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/bin/intel64/vc14/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
         COMMENT "Move tbb shared libraries to ngraph build directory"
         DEPENDEES download
         )
@@ -64,7 +64,7 @@ if (WIN32)
     ExternalProject_Add_Step(
         ext_tbb
         CopyTBBIMP
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/lib/intel64/vc14/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/lib/intel64/vc14/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}
         COMMENT "Move tbb libraries to ngraph build directory"
         DEPENDEES download
         )
@@ -78,7 +78,7 @@ elseif(APPLE)
     )
 
     add_custom_command(TARGET ext_tbb POST_BUILD
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INSTALL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
         COMMENT "Move tbb libraries to ngraph build directory"
     )
 

cmake/mkldnn.patch

 diff --git a/cmake/OpenMP.cmake b/cmake/OpenMP.cmake
-index de9b24d7..766a2e7f 100644
+index 71ff59ae..aea7d3fa 100644
 --- a/cmake/OpenMP.cmake
 +++ b/cmake/OpenMP.cmake
-@@ -30,14 +30,18 @@ if (APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+@@ -31,7 +31,7 @@ if (APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
      # But we still want to build the library.
      set(_omp_severity "WARNING")
  else()
@@ -10,34 +10,21 @@ index de9b24d7..766a2e7f 100644
 +    set(_omp_severity "WARNING")
  endif()
  
- 
  macro(forbid_link_compiler_omp_rt)
-     if (NOT WIN32)
--        set_if(OpenMP_C_FOUND CMAKE_C_CREATE_SHARED_LIBRARY_FORBIDDEN_FLAGS ${OpenMP_C_FLAGS})
--        set_if(OpenMP_CXX_FOUND CMAKE_CXX_CREATE_SHARED_LIBRARY_FORBIDDEN_FLAGS ${OpenMP_CXX_FLAGS})
-+        if (OpenMP_C_FOUND)
-+            set(CMAKE_C_CREATE_SHARED_LIBRARY_FORBIDDEN_FLAGS ${OpenMP_C_FLAGS})
-+        endif()
-+        if (OpenMP_CXX_FOUND)
-+            set(CMAKE_CXX_CREATE_SHARED_LIBRARY_FORBIDDEN_FLAGS ${OpenMP_CXX_FLAGS})
-+        endif()
-         if (NOT APPLE)
-             set (CMAKE_SHARED_LINKER_FLAGS "-Wl,--as-needed")
-         endif()
 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
-index d541d7ae..3d2b2599 100644
+index f10feb20..05f47961 100644
 --- a/src/CMakeLists.txt
 +++ b/src/CMakeLists.txt
-@@ -168,8 +168,10 @@ target_link_libraries(
- 
- set_property(TARGET ${TARGET_NAME} PROPERTY CXX_STANDARD 11)
- set_property(TARGET ${TARGET_NAME} PROPERTY CXX_STANDARD_REQUIRED ON)
--set_property(TARGET ${TARGET_NAME} PROPERTY VERSION "${PROJECT_VERSION}.0")
--set_property(TARGET ${TARGET_NAME} PROPERTY SOVERSION "0")
+@@ -92,8 +92,10 @@ endif()
+ add_library(${LIB_NAME} ${MKLDNN_LIBRARY_TYPE} ${HEADERS} ${SOURCES})
+ set_property(TARGET ${LIB_NAME} PROPERTY CXX_STANDARD 11)
+ set_property(TARGET ${LIB_NAME} PROPERTY CXX_STANDARD_REQUIRED ON)
+-set_property(TARGET ${LIB_NAME} PROPERTY VERSION "${PROJECT_VERSION}.0")
+-set_property(TARGET ${LIB_NAME} PROPERTY SOVERSION "0")
 +if(MKLDNN_LIB_VERSIONING_ENABLE)
-+    set_property(TARGET ${TARGET_NAME} PROPERTY VERSION "${PROJECT_VERSION}.0")
-+    set_property(TARGET ${TARGET_NAME} PROPERTY SOVERSION "0")
++    set_property(TARGET ${LIB_NAME} PROPERTY VERSION "${PROJECT_VERSION}.0")
++    set_property(TARGET ${LIB_NAME} PROPERTY SOVERSION "0")
 +endif()
+ set_property(TARGET ${LIB_NAME} PROPERTY PUBLIC_HEADER ${HEADERS})
  
- install(TARGETS ${TARGET_NAME}
-     EXPORT "${TARGETS_EXPORT_NAME}"
+ target_include_directories(${LIB_NAME} PUBLIC

src/ngraph/CMakeLists.txt

@@ -270,6 +270,15 @@ if (NGRAPH_ONNX_IMPORT_ENABLE)
     target_link_libraries(ngraph PRIVATE onnx_import)
 endif()
 
+# Build subdirectories for all build types on Windows
+if(WIN32)
+    foreach(BUILD_TYPE Release Debug RelWithDebInfo MinSizeRel)
+        if(NOT EXISTS ${NGRAPH_BUILD_DIR}/${BUILD_TYPE})
+            file(MAKE_DIRECTORY ${NGRAPH_BUILD_DIR}/${BUILD_TYPE})
+        endif()
+    endforeach()
+endif()
+
 #-----------------------------------------------------------------------------------------------
 # Installation logic...
 #-----------------------------------------------------------------------------------------------

src/ngraph/codegen/CMakeLists.txt

@@ -31,7 +31,9 @@ add_library(codegen SHARED ${SRC})
 # LLVM binary builds are typically built without RTTI
 # The built-in headers are in a version-specific directory
 # This must be kept in sync with the LLVM + Clang version in use
-set_source_files_properties(compiler.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti")
+if(NOT WIN32)
+   set_source_files_properties(compiler.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti")
+endif()
 
 get_target_property(LLVM_INCLUDE_DIR libllvm INTERFACE_INCLUDE_DIRECTORIES)
 

src/ngraph/file_util.cpp

@@ -306,7 +306,24 @@ void file_util::iterate_files(const string& path,
     {
         do
         {
-            func(data.cFileName, (data.dwFileAttributes == FILE_ATTRIBUTE_DIRECTORY));
+            bool is_dir = data.dwFileAttributes == FILE_ATTRIBUTE_DIRECTORY;
+            if (is_dir)
+            {
+                if (string(data.cFileName) != "." && string(data.cFileName) != "..")
+                {
+                    string dir_path = path_join(path, data.cFileName);
+                    if (recurse)
+                    {
+                        iterate_files(dir_path, func, recurse);
+                    }
+                    func(dir_path, true);
+                }
+            }
+            else
+            {
+                string file_name = path_join(path, data.cFileName);
+                func(file_name, false);
+            }
         } while (FindNextFile(hFind, &data));
         FindClose(hFind);
     }

src/ngraph/runtime/cpu/cpu_external_function.cpp

@@ -219,7 +219,8 @@ runtime::cpu::CPU_ExternalFunction::CPU_ExternalFunction(
     , m_use_tbb(std::getenv("NGRAPH_CPU_USE_TBB") != nullptr)
 #if !defined(NGRAPH_DEX_ONLY)
     , m_is_compiled(false)
-    , m_direct_execution(!std::getenv("NGRAPH_CODEGEN"))
+    , m_direct_execution((std::getenv("NGRAPH_CODEGEN") == nullptr) ||
+                         (std::string(std::getenv("NGRAPH_CODEGEN")) == "0"))
 #else
     , m_direct_execution(true)
 #endif

src/ngraph/runtime/cpu/pass/cpu_rnn_fusion.hpp

@@ -66,7 +66,8 @@ private:
     void construct_rnn_lstm_fprop();
 };
 
-class ngraph::runtime::cpu::pass::MultiLayerRNNFusion : public ngraph::pass::RecurrentGraphRewrite
+class CPU_BACKEND_API ngraph::runtime::cpu::pass::MultiLayerRNNFusion
+    : public ngraph::pass::RecurrentGraphRewrite
 {
 public:
     MultiLayerRNNFusion()
@@ -79,7 +80,8 @@ private:
     void construct_multi_layer_rnn_fusion_fprop();
 };
 
-class ngraph::runtime::cpu::pass::BiDirectionalRnn : public ngraph::pass::GraphRewrite
+class CPU_BACKEND_API ngraph::runtime::cpu::pass::BiDirectionalRnn
+    : public ngraph::pass::GraphRewrite
 {
 public:
     BiDirectionalRnn()

src/resource/main.cpp

@@ -110,7 +110,6 @@ int main(int argc, char** argv)
 
     for (ResourceInfo& path : include_paths)
     {
-        // cout << "path " << path.source_path << " -> " << path.target_path << endl;
         vector<string> path_list;
         if (path.subdirs.empty())
         {
@@ -131,7 +130,7 @@ int main(int argc, char** argv)
                                       string ext = get_file_ext(file);
                                       if (contains(valid_ext, ext))
                                       {
-                                          //   cout << "add " << path.search_path << ", " << file << endl;
+                                          // std::cout << "add " << file << std::endl;
                                           path.files.push_back(file);
                                       }
                                   }

src/resource/util.cpp

@@ -14,7 +14,11 @@
 // limitations under the License.
 //*****************************************************************************
 
+#ifdef _WIN32
+#include <windows.h>
+#else
 #include <dirent.h>
+#endif
 #include <fcntl.h>
 #include <fstream>
 #include <iomanip>
@@ -126,6 +130,7 @@ std::string read_file_to_string(const std::string& path)
     return ss.str();
 }
 
+#ifndef _WIN32
 void iterate_files_worker(const string& path,
                           std::function<void(const string& file, bool is_dir)> func,
                           bool recurse)
@@ -165,6 +170,7 @@ void iterate_files_worker(const string& path,
         closedir(dir);
     }
 }
+#endif
 
 void iterate_files(const string& path,
                    std::function<void(const string& file, bool is_dir)> func,
@@ -172,6 +178,36 @@ void iterate_files(const string& path,
 {
     vector<string> files;
     vector<string> dirs;
+#ifdef _WIN32
+    string file_match = path_join(path, "*");
+    WIN32_FIND_DATA data;
+    HANDLE hFind = FindFirstFile(file_match.c_str(), &data);
+    if (hFind != INVALID_HANDLE_VALUE)
+    {
+        do
+        {
+            bool is_dir = data.dwFileAttributes == FILE_ATTRIBUTE_DIRECTORY;
+            if (is_dir)
+            {
+                if (string(data.cFileName) != "." && string(data.cFileName) != "..")
+                {
+                    string dir_path = path_join(path, data.cFileName);
+                    if (recurse)
+                    {
+                        iterate_files(dir_path, func, recurse);
+                    }
+                    func(dir_path, true);
+                }
+            }
+            else
+            {
+                string file_name = path_join(path, data.cFileName);
+                func(file_name, false);
+            }
+        } while (FindNextFile(hFind, &data));
+        FindClose(hFind);
+    }
+#else
     iterate_files_worker(path,
                          [&files, &dirs](const string& file, bool is_dir) {
                              if (is_dir)
@@ -180,7 +216,7 @@ void iterate_files(const string& path,
                                  files.push_back(file);
                          },
                          recurse);
-
+#endif
     for (auto f : files)
     {
         func(f, false);