Embed header files into ngraph (#323)

* add resource file generator and store all headers used by codegen in memory.

src/CMakeLists.txt

@@ -13,5 +13,5 @@
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DIN_NGRAPH_LIBRARY")
 
-add_subdirectory( ngraph )
-
+add_subdirectory(resource)
+add_subdirectory(ngraph)

src/ngraph/CMakeLists.txt

@@ -170,6 +170,16 @@ if (NGRAPH_CPU_ENABLE AND LLVM_INCLUDE_DIR AND
 endif()
 
 add_library(ngraph SHARED ${SRC})
+if (NGRAPH_CPU_ENABLE AND LLVM_INCLUDE_DIR AND MKLDNN_INCLUDE_DIR)
+    # Generate the resource file containing all headers used by the codegen compiler
+    add_custom_target(header_resource
+        resource_generator --output ${CMAKE_BINARY_DIR}/header_resource.hpp --base codegen
+        DEPENDS resource_generator eigen ext_llvm ext_mkldnn
+        BYPRODUCTS
+    )
+    add_dependencies(ngraph header_resource)
+    include_directories(${CMAKE_BINARY_DIR})
+endif()
 
 if (NOT APPLE)
     # NGraph links against one or more libraries (ex. LLVM) but we don't want to
@@ -234,48 +244,9 @@ install(DIRECTORY
     FILES_MATCHING PATTERN "*.hpp"
 )
 
-# External
-# Requirements:
-# - In NGraph, there are multiple include paths set for different external projects.
-# - However, we want one single include path for external projects from the TF bridge side, while
-#   making sure that the `#include` files can still be resolved.
-# - Therefore, when `make install`, the include directory will be "flattened" for each external
-#   projects respectively.
-
-install(DIRECTORY
-    ${EIGEN_INCLUDE_DIR}/
-    DESTINATION "${NGRAPH_INSTALL_INCLUDE}"
-)
 if (NOT APPLE)
-install(DIRECTORY
-    ${MKLDNN_INCLUDE_DIR}/
-    DESTINATION "${NGRAPH_INSTALL_INCLUDE}"
-)
 install(DIRECTORY
     ${MKLDNN_LIB_DIR}/
     DESTINATION "${NGRAPH_INSTALL_LIB}"
 )
 endif()
-if (NGRAPH_CPU_ENABLE AND LLVM_INCLUDE_DIR AND
-    MKLDNN_INCLUDE_DIR)
-    # Currently this is required for Argon codegen. To avoid this, we'll need to decouple LLVM from
-    # compiler.hpp
-    install(DIRECTORY
-        ${LLVM_INCLUDE_DIR}/
-        DESTINATION "${NGRAPH_INSTALL_INCLUDE}"
-    )
-    install(DIRECTORY
-        ${LLVM_LIB_DIR}/
-        DESTINATION "${NGRAPH_INSTALL_LIB}"
-)
-endif()
-
-add_dependencies(ngraph eigen)
-
-if((NGRAPH_CPU_ENABLE OR NGRAPH_GPU_ENABLE) AND LLVM_INCLUDE_DIR)
-    add_dependencies(ngraph ext_llvm)
-endif()
-
-if(NGRAPH_CPU_ENABLE AND MKLDNN_INCLUDE_DIR)
-    add_dependencies(ngraph ext_mkldnn)
-endif()

src/ngraph/codegen/compiler.cpp

@@ -42,6 +42,7 @@
 #include <llvm/Support/Timer.h>
 #include <llvm/Support/raw_ostream.h>
 
+#include "header_resource.hpp"
 #include "ngraph/codegen/compiler.hpp"
 #include "ngraph/file_util.hpp"
 #include "ngraph/log.hpp"
@@ -60,16 +61,14 @@
 #error "This source file interfaces with LLVM and Clang and must be compiled with RTTI disabled"
 #endif
 
-// #define USE_CACHE
+#define USE_BUILTIN
 
 using namespace clang;
 using namespace llvm;
 using namespace llvm::opt;
 using namespace std;
-
 using namespace ngraph::codegen;
 
-static HeaderCache s_header_cache;
 static StaticCompiler s_static_compiler;
 static std::mutex m_mutex;
 
@@ -151,55 +150,7 @@ StaticCompiler::StaticCompiler()
         m_compiler->getHeaderSearchOpts().ResourceDir = path;
     }
 
-    if (s_header_cache.is_valid() == false)
-    {
-        // Add base toolchain-supplied header paths
-        // Ideally one would use the Linux toolchain definition in clang/lib/Driver/ToolChains.h
-        // But that's a private header and isn't part of the public libclang API
-        // Instead of re-implementing all of that functionality in a custom toolchain
-        // just hardcode the paths relevant to frequently used build/test machines for now
-        add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
-        add_header_search_path("/usr/include/x86_64-linux-gnu");
-        add_header_search_path("/usr/include");
-
-        // Search for headers in
-        //    /usr/include/x86_64-linux-gnu/c++/N.N
-        //    /usr/include/c++/N.N
-        // and add them to the header search path
-
-        file_util::iterate_files("/usr/include/x86_64-linux-gnu/c++/",
-                                 [&](const std::string& file, bool is_dir) {
-                                     if (is_dir)
-                                     {
-                                         string dir_name = file_util::get_file_name(file);
-                                         if (is_version_number(dir_name))
-                                         {
-                                             add_header_search_path(file);
-                                         }
-                                     }
-                                 });
-
-        file_util::iterate_files("/usr/include/c++/", [&](const std::string& file, bool is_dir) {
-            if (is_dir)
-            {
-                string dir_name = file_util::get_file_name(file);
-                if (is_version_number(dir_name))
-                {
-                    add_header_search_path(file);
-                }
-            }
-        });
-
-        add_header_search_path(EIGEN_HEADERS_PATH);
-        add_header_search_path(NGRAPH_HEADERS_PATH);
-#ifdef USE_CACHE
-        s_header_cache.set_valid();
-#endif
-    }
-
-#ifdef USE_CACHE
-    use_cached_files(m_compiler);
-#endif
+    configure_search_path();
 
     // Language options
     // These are the C++ features needed to compile ngraph headers
@@ -276,49 +227,7 @@ void StaticCompiler::add_header_search_path(const string& path)
     {
         m_extra_search_path_list.push_back(path);
         HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
-#ifdef USE_CACHE
-        static vector<string> valid_ext = {".h", ".hpp", ".tcc", ""};
-        string mapped_path = file_util::path_join("/$BUILTIN", path);
-        mapped_path = path;
-        s_header_cache.add_path(mapped_path);
-        auto func = [&](const std::string& file, bool is_dir) {
-            if (!is_dir)
-            {
-                string ext = file_util::get_file_ext(file);
-                if (contains(valid_ext, ext))
-                {
-                    // This is a header file
-                    string relative_name = file.substr(path.size() + 1);
-                    string mapped_name = file_util::path_join(mapped_path, relative_name);
-
-                    ErrorOr<unique_ptr<MemoryBuffer>> code = MemoryBuffer::getFile(file);
-                    if (error_code ec = code.getError())
-                    {
-                        // throw up
-                        throw runtime_error("could not find file '" + file + "'");
-                    }
-
-                    s_header_cache.add_file(mapped_name, code.get());
-                }
-            }
-        };
-        file_util::iterate_files(path, func, true);
-#else
         hso.AddPath(path, clang::frontend::System, false, false);
-#endif
-    }
-}
-
-void StaticCompiler::use_cached_files()
-{
-    HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
-    for (const string& path : s_header_cache.get_include_paths())
-    {
-        hso.AddPath(path, clang::frontend::System, false, false);
-    }
-    for (auto& header : s_header_cache.get_header_map())
-    {
-        m_compiler->getPreprocessorOpts().addRemappedFile(header.first, header.second.get());
     }
 }
 
@@ -380,3 +289,82 @@ void StaticCompiler::generate_pch(const string& source)
     m_compiler->getInvocation().getPreprocessorOpts().clearRemappedFiles();
     delete compilerAction;
 }
+
+void StaticCompiler::configure_search_path()
+{
+#ifdef USE_BUILTIN
+    load_header_search_path_from_resource();
+    load_headers_from_resource();
+#else
+    // Add base toolchain-supplied header paths
+    // Ideally one would use the Linux toolchain definition in clang/lib/Driver/ToolChains.h
+    // But that's a private header and isn't part of the public libclang API
+    // Instead of re-implementing all of that functionality in a custom toolchain
+    // just hardcode the paths relevant to frequently used build/test machines for now
+    add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
+    add_header_search_path("/usr/include/x86_64-linux-gnu");
+    add_header_search_path("/usr/include");
+
+    // Search for headers in
+    //    /usr/include/x86_64-linux-gnu/c++/N.N
+    //    /usr/include/c++/N.N
+    // and add them to the header search path
+
+    file_util::iterate_files("/usr/include/x86_64-linux-gnu/c++/",
+                             [&](const std::string& file, bool is_dir) {
+                                 if (is_dir)
+                                 {
+                                     string dir_name = file_util::get_file_name(file);
+                                     if (is_version_number(dir_name))
+                                     {
+                                         add_header_search_path(file);
+                                     }
+                                 }
+                             });
+
+    file_util::iterate_files("/usr/include/c++/", [&](const std::string& file, bool is_dir) {
+        if (is_dir)
+        {
+            string dir_name = file_util::get_file_name(file);
+            if (is_version_number(dir_name))
+            {
+                add_header_search_path(file);
+            }
+        }
+    });
+
+    add_header_search_path(EIGEN_HEADERS_PATH);
+    add_header_search_path(NGRAPH_HEADERS_PATH);
+#endif
+}
+
+void StaticCompiler::load_header_search_path_from_resource()
+{
+    HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
+
+    std::vector<std::string> header_search_paths;
+    for (const HeaderInfo& hi : header_info)
+    {
+        string search_path = hi.search_path;
+        if (!contains(header_search_paths, search_path))
+        {
+            string builtin = "/$builtin" + search_path;
+            hso.AddPath(builtin, clang::frontend::System, false, false);
+            header_search_paths.push_back(search_path);
+        }
+    }
+}
+
+void StaticCompiler::load_headers_from_resource()
+{
+    HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
+    for (const HeaderInfo& hi : header_info)
+    {
+        string search_path = hi.search_path;
+        string absolute_path = file_util::path_join(search_path, hi.header_path);
+        string builtin = "/$builtin" + absolute_path;
+        std::unique_ptr<llvm::MemoryBuffer> mb(
+            llvm::MemoryBuffer::getMemBuffer(hi.header_data, builtin));
+        m_compiler->getPreprocessorOpts().addRemappedFile(builtin, mb.release());
+    }
+}

src/ngraph/codegen/compiler.hpp

@@ -89,26 +89,7 @@ private:
     std::string m_precomiled_header_source;
 
     bool is_version_number(const std::string& path);
-    void use_cached_files();
-};
-
-class ngraph::codegen::HeaderCache
-{
-public:
-    bool is_valid() const { return m_headers_valid; }
-    bool set_valid() { return m_headers_valid = true; }
-    void add_path(const std::string& path) { m_include_paths.push_back(path); }
-    void add_file(const std::string& path, std::unique_ptr<llvm::MemoryBuffer>& code)
-    {
-        m_headers.insert(std::make_pair(path, std::move(code)));
-    }
-    const std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>>& get_header_map() const
-    {
-        return m_headers;
-    }
-    const std::vector<std::string>& get_include_paths() const { return m_include_paths; }
-private:
-    std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>> m_headers;
-    std::vector<std::string> m_include_paths;
-    bool m_headers_valid;
+    void configure_search_path();
+    void load_header_search_path_from_resource();
+    void load_headers_from_resource();
 };

src/ngraph/file_util.cpp

@@ -331,3 +331,14 @@ void ngraph::file_util::release_lock(int fd, const std::string& filename)
         close(fd);
     }
 }
+
+time_t ngraph::file_util::get_timestamp(const std::string& filename)
+{
+    time_t rc = 0;
+    struct stat st;
+    if (stat(filename.c_str(), &st) == 0)
+    {
+        rc = st.st_mtime;
+    }
+    return rc;
+}

src/ngraph/file_util.hpp

@@ -45,6 +45,7 @@ public:
     static bool exists(const std::string& filename);
     static int try_get_lock(const std::string& filename);
     static void release_lock(int fd, const std::string& filename);
+    static time_t get_timestamp(const std::string& filename);
 
 private:
     static void iterate_files_worker(const std::string& path,

src/resource/CMakeLists.txt

+# Copyright 2017 Nervana Systems Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set (SRC
+    main.cpp
+    util.cpp
+    uncomment.cpp
+    header_rewrite.cpp
+)
+
+add_executable(resource_generator ${SRC})
+
+set_source_files_properties(main.cpp PROPERTIES COMPILE_DEFINITIONS
+    "EIGEN_HEADERS_PATH=\"${EIGEN_INCLUDE_DIR}\";CLANG_BUILTIN_HEADERS_PATH=\"${LLVM_LIB_DIR}/clang/5.0.0/include\";NGRAPH_HEADERS_PATH=\"${NGRAPH_INCLUDE_PATH}\";")

src/resource/header_rewrite.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <sstream>
+#include <vector>
+
+#include "header_rewrite.hpp"
+#include "util.hpp"
+
+using namespace std;
+
+// This function rewrites all of the
+// #include "../../blah"
+// into something with a dotless relative path. It seems that clang can't handle the .. stuff
+// when the header files are stored in its in-memory filesystem.
+// Eigen has a lot of .. in their header files.
+const string rewrite_header(const string& s, const string& path)
+{
+    stringstream ss(s);
+    stringstream out;
+    for (string line; ss; getline(ss, line))
+    {
+        string original_line = line;
+        // only interested in lines starging with '#include' so 8 chars minimum
+        if (line.size() > 8)
+        {
+            // skip whitespace
+            size_t pos = line.find_first_not_of(" \t");
+            if (pos != string::npos && line[pos] == '#' && pos < line.size() - 7)
+            {
+                string directive = line;
+                pos = directive.find_first_not_of(" \t", pos + 1);
+                if (pos != string::npos)
+                {
+                    directive = directive.substr(pos);
+                }
+                pos = directive.find_first_of(" \t", pos + 1);
+                directive = directive.substr(0, pos);
+                if (directive == "include")
+                {
+                    auto line_offset = line.find_first_of("\"<");
+                    if (line_offset != string::npos)
+                    {
+                        string include = line.substr(line_offset);
+                        string contents = include.substr(1, include.size() - 2);
+                        if (include[1] == '.')
+                        {
+                            if (include[2] == '/')
+                            {
+                                // include starts with './'
+                                // rewrite "./blah.h" to "blah.h"
+                                contents = contents.substr(2);
+                            }
+                            else
+                            {
+                                // include starts with '../'
+                                // count number of '../' in string
+                                size_t offset = 0;
+                                size_t depth = 0;
+                                while (contents.substr(offset, 3) == "../")
+                                {
+                                    depth++;
+                                    offset += 3;
+                                }
+                                string trimmed = contents.substr(offset);
+                                vector<string> parts = split(path, '/');
+                                parts.pop_back();
+                                size_t result_depth = parts.size() - depth;
+                                string added_path;
+                                for (size_t i = 0; i < result_depth; i++)
+                                {
+                                    added_path += parts[i] + "/";
+                                }
+                                contents = added_path + trimmed;
+                            }
+                            if (include[0] == '<')
+                            {
+                                line = "#include <" + contents + ">";
+                            }
+                            else
+                            {
+                                line = "#include \"" + contents + "\"";
+                            }
+                            // cout << "line '" << original_line << "'\n";
+                            // cout << "rewrite to '" << line << "'\n\n";
+                        }
+                    }
+                }
+            }
+        }
+        out << line << "\n";
+    }
+    return out.str();
+}

src/resource/header_rewrite.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <string>
+
+const std::string rewrite_header(const std::string& s, const std::string& path);

src/resource/main.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "header_rewrite.hpp"
+#include "uncomment.hpp"
+#include "util.hpp"
+
+using namespace std;
+
+class ResourceInfo
+{
+public:
+    ResourceInfo(const string& source, const vector<string>& _subdirs, bool recursive = false)
+        : search_path(source)
+        , subdirs(_subdirs)
+        , is_recursive(recursive)
+
+    {
+    }
+
+    const string search_path;
+    const vector<string> subdirs;
+    const bool is_recursive;
+
+    vector<string> files;
+};
+
+string find_path(const string& path)
+{
+    string rc;
+    iterate_files(path,
+                  [&](const string& file, bool is_dir) {
+                      if (is_dir)
+                      {
+                          string dir_name = get_file_name(file);
+                          if (is_version_number(dir_name))
+                          {
+                              rc = file;
+                          }
+                      }
+                  },
+                  true);
+    return rc;
+}
+
+int main(int argc, char** argv)
+{
+    time_t main_timestamp = get_timestamp(argv[0]);
+    static vector<string> valid_ext = {".h", ".hpp", ".tcc", ""};
+    string output_path;
+    string base_name;
+
+    for (size_t i = 1; i < argc; i++)
+    {
+        if (string(argv[i]) == "--output")
+        {
+            output_path = argv[++i];
+        }
+        else if (string(argv[i]) == "--base_name")
+        {
+            base_name = argv[++i];
+        }
+    }
+
+    string cpp0 = find_path("/usr/include/x86_64-linux-gnu/c++/");
+    string cpp1 = find_path("/usr/include/c++/");
+
+    vector<ResourceInfo> include_paths;
+    include_paths.push_back({CLANG_BUILTIN_HEADERS_PATH, {}, true});
+    include_paths.push_back({"/usr/include/x86_64-linux-gnu", {"asm", "sys", "bits", "gnu"}});
+    include_paths.push_back({"/usr/include", {"linux", "asm-generic"}});
+    include_paths.push_back({cpp0, {"bits"}});
+    include_paths.push_back({cpp1, {"bits", "ext", "debug", "backward"}});
+    include_paths.push_back({EIGEN_HEADERS_PATH, {}, true});
+    include_paths.push_back({NGRAPH_HEADERS_PATH, {}, true});
+
+    if (output_path.empty())
+    {
+        cout << "must specify output path with --output option" << endl;
+        return -1;
+    }
+
+    time_t output_timestamp = get_timestamp(output_path);
+
+    for (ResourceInfo& path : include_paths)
+    {
+        // cout << "path " << path.source_path << " -> " << path.target_path << endl;
+        vector<string> path_list;
+        path_list.push_back(path.search_path);
+        for (const string& p : path.subdirs)
+        {
+            path_list.push_back(path_join(path.search_path, p));
+        }
+        for (const string& p : path_list)
+        {
+            iterate_files(p,
+                          [&](const string& file, bool is_dir) {
+                              if (!is_dir)
+                              {
+                                  string ext = get_file_ext(file);
+                                  if (contains(valid_ext, ext))
+                                  {
+                                      //   cout << "add " << path.search_path << ", " << file << endl;
+                                      path.files.push_back(file);
+                                  }
+                              }
+                          },
+                          path.is_recursive);
+        }
+    }
+
+    // test for changes to any headers
+    bool update_needed = main_timestamp > output_timestamp;
+    if (!update_needed)
+    {
+        for (ResourceInfo& path : include_paths)
+        {
+            for (const string& header_file : path.files)
+            {
+                time_t file_timestamp = get_timestamp(header_file);
+                if (file_timestamp > output_timestamp)
+                {
+                    update_needed = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (update_needed)
+    {
+        ofstream out(output_path);
+        out << "#pragma clang diagnostic ignored \"-Weverything\"\n";
+        out << "#include <vector>\n";
+        out << "namespace ngraph\n";
+        out << "{\n";
+        out << "    uint8_t header_resources[] =\n";
+        out << "    {\n";
+        vector<pair<size_t, size_t>> offset_size_list;
+        size_t offset = 0;
+        size_t total_size = 0;
+        size_t total_count = 0;
+        for (const ResourceInfo& path : include_paths)
+        {
+            for (const string& header_file : path.files)
+            {
+                string header_data = read_file_to_string(header_file);
+                string base_path = header_file.substr(path.search_path.size() + 1);
+                header_data = rewrite_header(header_data, base_path);
+                // header_data = uncomment(header_data);
+                total_size += header_data.size();
+                total_count++;
+
+                // data layout is triplet of strings containing:
+                // 1) search path
+                // 2) header path within search path
+                // 3) header data
+                // all strings are null terminated and the length includes the null
+                // The + 1 below is to account for the null terminator
+                dump(out, path.search_path.c_str(), path.search_path.size() + 1);
+                offset_size_list.push_back({offset, path.search_path.size() + 1});
+                offset += path.search_path.size() + 1;
+
+                dump(out, header_file.c_str(), header_file.size() + 1);
+                offset_size_list.push_back({offset, header_file.size() + 1});
+                offset += header_file.size() + 1;
+
+                dump(out, header_data.c_str(), header_data.size() + 1);
+                offset_size_list.push_back({offset, header_data.size() + 1});
+                offset += header_data.size() + 1;
+            }
+        }
+        out << "    };\n";
+        out << "    struct HeaderInfo\n";
+        out << "    {\n";
+        out << "        const char* search_path;\n";
+        out << "        const char* header_path;\n";
+        out << "        const char* header_data;\n";
+        out << "    };\n";
+        out << "    std::vector<HeaderInfo> header_info\n";
+        out << "    {\n";
+        for (size_t i = 0; i < offset_size_list.size();)
+        {
+            out << "        {(char*)(&header_resources[" << offset_size_list[i++].first;
+            out << "]), (char*)(&header_resources[" << offset_size_list[i++].first;
+            out << "]), (char*)(&header_resources[" << offset_size_list[i++].first << "])},\n";
+        }
+        out << "    };\n";
+        out << "}\n";
+        cout.imbue(locale(""));
+        cout << "Total size " << total_size << " in " << total_count << " files\n";
+    }
+}

src/resource/uncomment.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <iostream>
+#include <sstream>
+
+#include "uncomment.hpp"
+
+using namespace std;
+
+// start 23,749,645 in 1,912 files
+
+void skip_comment(istream& s)
+{
+}
+
+string uncomment(const string& s)
+{
+    stringstream out;
+    for (size_t i = 0; i < s.size(); i++)
+    {
+        char c = s[i];
+        if (i < s.size() - 1 && c == '/' && s[i + 1] == '/')
+        {
+            while (i < s.size() && c != '\n')
+            {
+                c = s[++i];
+            }
+            out << "\n";
+        }
+        else
+        {
+            out << c;
+        }
+    }
+    return out.str();
+}

src/resource/uncomment.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <string>
+
+std::string uncomment(const std::string& s);

src/resource/util.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <fstream>
+#include <iomanip>
+#include <sstream>
+
+#include "util.hpp"
+
+using namespace std;
+
+string trim(const string& s)
+{
+    string rc = s;
+    // trim trailing spaces
+    size_t pos = rc.find_last_not_of(" \t");
+    if (string::npos != pos)
+    {
+        rc = rc.substr(0, pos + 1);
+    }
+
+    // trim leading spaces
+    pos = rc.find_first_not_of(" \t");
+    if (string::npos != pos)
+    {
+        rc = rc.substr(pos);
+    }
+    return rc;
+}
+
+vector<string> split(const string& src, char delimiter, bool do_trim)
+{
+    size_t pos;
+    string token;
+    size_t start = 0;
+    vector<string> rc;
+    while ((pos = src.find(delimiter, start)) != std::string::npos)
+    {
+        token = src.substr(start, pos - start);
+        start = pos + 1;
+        if (do_trim)
+        {
+            token = trim(token);
+        }
+        rc.push_back(token);
+    }
+    if (start <= src.size())
+    {
+        token = src.substr(start);
+        if (do_trim)
+        {
+            token = trim(token);
+        }
+        rc.push_back(token);
+    }
+    return rc;
+}
+
+bool is_version_number(const string& path)
+{
+    bool rc = true;
+    vector<string> tokens = split(path, '.');
+    for (string s : tokens)
+    {
+        for (char c : s)
+        {
+            if (!isdigit(c))
+            {
+                rc = false;
+            }
+        }
+    }
+    return rc;
+}
+
+string path_join(const string& s1, const string& s2)
+{
+    string rc;
+    if (s2.size() > 0)
+    {
+        if (s2[0] == '/')
+        {
+            rc = s2;
+        }
+        else if (s1.size() > 0)
+        {
+            rc = s1;
+            if (rc[rc.size() - 1] != '/')
+            {
+                rc += "/";
+            }
+            rc += s2;
+        }
+        else
+        {
+            rc = s2;
+        }
+    }
+    else
+    {
+        rc = s1;
+    }
+    return rc;
+}
+
+std::string read_file_to_string(const std::string& path)
+{
+    std::ifstream f(path);
+    std::stringstream ss;
+    ss << f.rdbuf();
+    return ss.str();
+}
+
+void iterate_files_worker(const string& path,
+                          std::function<void(const string& file, bool is_dir)> func,
+                          bool recurse)
+{
+    DIR* dir;
+    struct dirent* ent;
+    if ((dir = opendir(path.c_str())) != nullptr)
+    {
+        while ((ent = readdir(dir)) != nullptr)
+        {
+            string name = ent->d_name;
+            switch (ent->d_type)
+            {
+            case DT_DIR:
+                if (name != "." && name != "..")
+                {
+                    string dir_path = path_join(path, name);
+                    if (recurse)
+                    {
+                        iterate_files(dir_path, func, recurse);
+                    }
+                    func(dir_path, true);
+                }
+                break;
+            case DT_LNK: break;
+            case DT_REG:
+            {
+                string file_name = path_join(path, name);
+                func(file_name, false);
+                break;
+            }
+            default: break;
+            }
+        }
+        closedir(dir);
+    }
+    else
+    {
+        throw std::runtime_error("error enumerating file " + path);
+    }
+}
+
+void iterate_files(const string& path,
+                   std::function<void(const string& file, bool is_dir)> func,
+                   bool recurse)
+{
+    vector<string> files;
+    vector<string> dirs;
+    iterate_files_worker(path,
+                         [&files, &dirs](const string& file, bool is_dir) {
+                             if (is_dir)
+                                 dirs.push_back(file);
+                             else
+                                 files.push_back(file);
+                         },
+                         recurse);
+
+    for (auto f : files)
+    {
+        func(f, false);
+    }
+    for (auto f : dirs)
+    {
+        func(f, true);
+    }
+}
+
+std::string get_file_name(const std::string& s)
+{
+    string rc = s;
+    auto pos = s.find_last_of('/');
+    if (pos != string::npos)
+    {
+        rc = s.substr(pos + 1);
+    }
+    return rc;
+}
+
+std::string get_file_ext(const std::string& s)
+{
+    string rc = get_file_name(s);
+    auto pos = rc.find_last_of('.');
+    if (pos != string::npos)
+    {
+        rc = rc.substr(pos);
+    }
+    else
+    {
+        rc = "";
+    }
+    return rc;
+}
+
+string to_hex(int value)
+{
+    stringstream ss;
+    ss << "0x" << std::hex << std::setw(2) << std::setfill('0') << value;
+    return ss.str();
+}
+
+void dump(ostream& out, const void* vdata, size_t size)
+{
+    const uint8_t* data = reinterpret_cast<const uint8_t*>(vdata);
+    size_t index = 0;
+    while (index < size)
+    {
+        out << "        ";
+        for (size_t i = 0; i < 16 && index < size; i++)
+        {
+            if (i != 0)
+            {
+                out << ", ";
+            }
+            out << to_hex(data[index++]);
+        }
+        out << ",\n";
+    }
+}
+
+time_t get_timestamp(const std::string& filename)
+{
+    time_t rc = 0;
+    struct stat st;
+    if (stat(filename.c_str(), &st) == 0)
+    {
+        rc = st.st_mtime;
+    }
+    return rc;
+}

src/resource/util.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <functional>
+#include <iostream>
+#include <string>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <vector>
+
+std::string trim(const std::string& s);
+std::vector<std::string> split(const std::string& src, char delimiter, bool do_trim = false);
+bool is_version_number(const std::string& path);
+std::string path_join(const std::string& s1, const std::string& s2);
+std::string read_file_to_string(const std::string& path);
+void iterate_files_worker(const std::string& path,
+                          std::function<void(const std::string& file, bool is_dir)> func,
+                          bool recurse);
+void iterate_files(const std::string& path,
+                   std::function<void(const std::string& file, bool is_dir)> func,
+                   bool recurse);
+std::string get_file_name(const std::string& s);
+std::string get_file_ext(const std::string& s);
+std::string to_hex(int value);
+void dump(std::ostream& out, const void* vdata, size_t size);
+time_t get_timestamp(const std::string& filename);
+
+template <typename U, typename T>
+bool contains(const U& container, const T& obj)
+{
+    bool rc = false;
+    for (auto o : container)
+    {
+        if (o == obj)
+        {
+            rc = true;
+            break;
+        }
+    }
+    return rc;
+}
+
+template <typename T>
+std::string join(const T& v, const std::string& sep = ", ")
+{
+    std::ostringstream ss;
+    for (const auto& x : v)
+    {
+        if (&x != &*(v.begin()))
+        {
+            ss << sep;
+        }
+        ss << x;
+    }
+    return ss.str();
+}