compiler update to search for header files and option to cache them

src/ngraph/codegen/compiler.cpp

@@ -49,9 +49,14 @@
 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
 
 #include "ngraph/codegen/compiler.hpp"
+#include "ngraph/file_util.hpp"
+#include "ngraph/log.hpp"
+#include "ngraph/util.hpp"
 
 // TODO: Fix leaks
 
+// #define USE_CACHE
+
 using namespace clang;
 using namespace llvm;
 using namespace llvm::opt;
@@ -59,6 +64,8 @@ using namespace std;
 
 using namespace ngraph::codegen;
 
+static HeaderCache s_header_cache;
+
 static std::string GetExecutablePath(const char* Argv0)
 {
     // This just needs to be some symbol in the binary; C++ doesn't
@@ -78,6 +85,70 @@ execution_state::~execution_state()
 {
 }
 
+bool execution_state::is_version_number(const string& path)
+{
+    bool rc = true;
+    vector<string> tokens = ngraph::split(path, '.');
+    for (string s : tokens)
+    {
+        for (char c : s)
+        {
+            if (!isdigit(c))
+            {
+                rc = false;
+            }
+        }
+    }
+    return rc;
+}
+
+void execution_state::add_header_search_path(HeaderSearchOptions& hso, const string& path)
+{
+    static vector<string> valid_ext = {".h", ".hpp", ".tcc", ""};
+
+#ifdef USE_CACHE
+    string mapped_path = file_util::path_join("/$BUILTIN", path);
+    mapped_path = path;
+    s_header_cache.add_path(mapped_path);
+    auto func = [&](const std::string& file, bool is_dir) {
+        if (!is_dir)
+        {
+            string ext = file_util::get_file_ext(file);
+            if (contains(valid_ext, ext))
+            {
+                // This is a header file
+                string relative_name = file.substr(path.size() + 1);
+                string mapped_name = file_util::path_join(mapped_path, relative_name);
+
+                ErrorOr<unique_ptr<MemoryBuffer>> code = MemoryBuffer::getFile(file);
+                if (error_code ec = code.getError())
+                {
+                    // throw up
+                }
+
+                s_header_cache.add_file(mapped_name, code.get());
+            }
+        }
+    };
+    file_util::iterate_files(path, func, true);
+#else
+    hso.AddPath(path, clang::frontend::System, false, false);
+#endif
+}
+
+void execution_state::use_cached_files(std::unique_ptr<CompilerInstance>& Clang)
+{
+    HeaderSearchOptions& hso = Clang->getInvocation().getHeaderSearchOpts();
+    for (const string& path : s_header_cache.get_include_paths())
+    {
+        hso.AddPath(path, clang::frontend::System, false, false);
+    }
+    for (auto& header : s_header_cache.get_header_map())
+    {
+        Clang->getPreprocessorOpts().addRemappedFile(header.first, header.second.get());
+    }
+}
+
 std::unique_ptr<llvm::Module> execution_state::compile(const string& source, const string& name)
 {
     llvm::InitializeAllTargets();
@@ -113,24 +184,56 @@ std::unique_ptr<llvm::Module> execution_state::compile(const string& source, con
         Clang->getHeaderSearchOpts().ResourceDir = path;
     }
 
-    auto& HSO = Clang->getInvocation().getHeaderSearchOpts();
-    // Add base toolchain-supplied header paths
-    // Ideally one would use the Linux toolchain definition in clang/lib/Driver/ToolChains.h
-    // But that's a private header and isn't part of the public libclang API
-    // Instead of re-implementing all of that functionality in a custom toolchain
-    // just hardcode the paths relevant to frequently used build/test machines for now
-    HSO.AddPath(CLANG_BUILTIN_HEADERS_PATH, clang::frontend::System, false, false);
-    HSO.AddPath("/usr/include/x86_64-linux-gnu", clang::frontend::System, false, false);
-    HSO.AddPath("/usr/include", clang::frontend::System, false, false);
-    // Add C++ standard library headers
-    // Debian-like + GCC 4.8 libstdc++
-    HSO.AddPath("/usr/include/x86_64-linux-gnu/c++/4.8", clang::frontend::System, false, false);
-    HSO.AddPath("/usr/include/c++/4.8", clang::frontend::System, false, false);
-    // Debian-like + GCC 5 libstdc++
-    HSO.AddPath("/usr/include/x86_64-linux-gnu/c++/5", clang::frontend::System, false, false);
-    HSO.AddPath("/usr/include/c++/5", clang::frontend::System, false, false);
-    HSO.AddPath(EIGEN_HEADERS_PATH, clang::frontend::System, false, false);
-    HSO.AddPath(NGRAPH_HEADERS_PATH, clang::frontend::System, false, false);
+    HeaderSearchOptions& hso = Clang->getInvocation().getHeaderSearchOpts();
+    if (s_header_cache.is_valid() == false)
+    {
+        // Add base toolchain-supplied header paths
+        // Ideally one would use the Linux toolchain definition in clang/lib/Driver/ToolChains.h
+        // But that's a private header and isn't part of the public libclang API
+        // Instead of re-implementing all of that functionality in a custom toolchain
+        // just hardcode the paths relevant to frequently used build/test machines for now
+        add_header_search_path(hso, CLANG_BUILTIN_HEADERS_PATH);
+        add_header_search_path(hso, "/usr/include/x86_64-linux-gnu");
+        add_header_search_path(hso, "/usr/include");
+
+        // Search for headers in
+        //    /usr/include/x86_64-linux-gnu/c++/N.N
+        //    /usr/include/c++/N.N
+        // and add them to the header search path
+
+        file_util::iterate_files("/usr/include/x86_64-linux-gnu/c++/",
+                                 [&](const std::string& file, bool is_dir) {
+                                     if (is_dir)
+                                     {
+                                         string dir_name = file_util::get_file_name(file);
+                                         if (is_version_number(dir_name))
+                                         {
+                                             add_header_search_path(hso, file);
+                                         }
+                                     }
+                                 });
+
+        file_util::iterate_files("/usr/include/c++/", [&](const std::string& file, bool is_dir) {
+            if (is_dir)
+            {
+                string dir_name = file_util::get_file_name(file);
+                if (is_version_number(dir_name))
+                {
+                    add_header_search_path(hso, file);
+                }
+            }
+        });
+
+        add_header_search_path(hso, EIGEN_HEADERS_PATH);
+        add_header_search_path(hso, NGRAPH_HEADERS_PATH);
+#ifdef USE_CACHE
+        s_header_cache.set_valid();
+#endif
+    }
+
+#ifdef USE_CACHE
+    use_cached_files(Clang);
+#endif
 
     // Language options
     // These are the C++ features needed to compile ngraph headers

src/ngraph/codegen/compiler.hpp

@@ -28,9 +28,16 @@ namespace ngraph
     {
         class module;
         class execution_state;
+        class HeaderCache;
     }
 }
 
+namespace clang
+{
+    class HeaderSearchOptions;
+    class CompilerInstance;
+}
+
 class ngraph::codegen::module
 {
 public:
@@ -73,4 +80,28 @@ private:
     {
         return static_cast<signature*>(reinterpret_cast<signature*>(f));
     }
+    bool is_version_number(const std::string& path);
+    void add_header_search_path(clang::HeaderSearchOptions& hso, const std::string& path);
+    void use_cached_files(std::unique_ptr<clang::CompilerInstance>& Clang);
+};
+
+class ngraph::codegen::HeaderCache
+{
+public:
+    bool is_valid() const { return m_headers_valid; }
+    bool set_valid() { return m_headers_valid = true; }
+    void add_path(const std::string& path) { m_include_paths.push_back(path); }
+    void add_file(const std::string& path, std::unique_ptr<llvm::MemoryBuffer>& code)
+    {
+        m_headers.insert(std::make_pair(path, std::move(code)));
+    }
+    const std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>>& get_header_map() const
+    {
+        return m_headers;
+    }
+    const std::vector<std::string>& get_include_paths() const { return m_include_paths; }
+private:
+    std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>> m_headers;
+    std::vector<std::string> m_include_paths;
+    bool m_headers_valid;
 };

src/ngraph/file_util.cpp

-/*
- Copyright 2016 Nervana Systems Inc.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
 
 #include <cassert>
 #include <dirent.h>
@@ -29,10 +28,37 @@
 #include <unistd.h>
 #include <vector>
 
-#include "file_util.hpp"
+#include "ngraph/file_util.hpp"
+#include "ngraph/log.hpp"
 
 using namespace std;
 
+std::string ngraph::file_util::get_file_name(const std::string& s)
+{
+    string rc = s;
+    auto pos = s.find_last_of('/');
+    if (pos != string::npos)
+    {
+        rc = s.substr(pos + 1);
+    }
+    return rc;
+}
+
+std::string ngraph::file_util::get_file_ext(const std::string& s)
+{
+    string rc = get_file_name(s);
+    auto pos = rc.find_last_of('.');
+    if (pos != string::npos)
+    {
+        rc = rc.substr(pos);
+    }
+    else
+    {
+        rc = "";
+    }
+    return rc;
+}
+
 string ngraph::file_util::path_join(const string& s1, const string& s2)
 {
     string rc;
@@ -79,20 +105,22 @@ size_t ngraph::file_util::get_file_size(const string& filename)
 void ngraph::file_util::remove_directory(const string& dir)
 {
     struct stat status;
-    if (stat(dir.c_str(), &status) == -1)
+    if (stat(dir.c_str(), &status) != -1)
     {
-        return;
+        iterate_files(dir,
+                      [](const string& file, bool is_dir) {
+                          if (is_dir)
+                          {
+                              rmdir(file.c_str());
+                          }
+                          else
+                          {
+                              remove(file.c_str());
+                          }
+                      },
+                      true);
+        rmdir(dir.c_str());
     }
-
-    file_util::iterate_files(dir,
-                             [](const string& file, bool is_dir) {
-                                 if (is_dir)
-                                     rmdir(file.c_str());
-                                 else
-                                     remove(file.c_str());
-                             },
-                             true);
-    rmdir(dir.c_str());
 }
 
 void ngraph::file_util::remove_file(const string& file)
@@ -129,7 +157,7 @@ string ngraph::file_util::make_temp_directory(const string& path)
 
 std::string ngraph::file_util::get_temp_directory()
 {
-    const vector<string> potential_tmps = {"NERVANA_AEON_TMP", "TMPDIR", "TMP", "TEMP", "TEMPDIR"};
+    const vector<string> potential_tmps = {"ngraph_AEON_TMP", "TMPDIR", "TMP", "TEMP", "TEMPDIR"};
 
     const char* path = nullptr;
     for (const string& var : potential_tmps)
@@ -159,9 +187,9 @@ vector<char> ngraph::file_util::read_file_contents(const string& path)
     if (f)
     {
         char* p = data.data();
-        size_t remainder = file_size;
+        int remainder = static_cast<int>(file_size);
         size_t offset = 0;
-        while (remainder > 0)
+        while (f && remainder > 0)
         {
             size_t rc = fread(&p[offset], 1, remainder, f);
             offset += rc;
@@ -197,12 +225,16 @@ void ngraph::file_util::iterate_files(const string& path,
                                         else
                                             files.push_back(file);
                                     },
-                                    true);
+                                    recurse);
 
     for (auto f : files)
+    {
         func(f, false);
+    }
     for (auto f : dirs)
+    {
         func(f, true);
+    }
 }
 
 void ngraph::file_util::iterate_files_worker(
@@ -218,18 +250,23 @@ void ngraph::file_util::iterate_files_worker(
             switch (ent->d_type)
             {
             case DT_DIR:
-                if (recurse && name != "." && name != "..")
+                if (name != "." && name != "..")
                 {
                     string dir_path = file_util::path_join(path, name);
-                    iterate_files(dir_path, func, recurse);
+                    if (recurse)
+                    {
+                        iterate_files(dir_path, func, recurse);
+                    }
                     func(dir_path, true);
                 }
                 break;
             case DT_LNK: break;
             case DT_REG:
-                name = file_util::path_join(path, name);
-                func(name, false);
+            {
+                string file_name = file_util::path_join(path, name);
+                func(file_name, false);
                 break;
+            }
             default: break;
             }
         }

src/ngraph/file_util.hpp

-/*
- Copyright 2016 Nervana Systems Inc.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
 
 #pragma once
 
@@ -27,6 +26,8 @@ namespace ngraph
 class ngraph::file_util
 {
 public:
+    static std::string get_file_name(const std::string&);
+    static std::string get_file_ext(const std::string&);
     static std::string path_join(const std::string& s1, const std::string& s2);
     static size_t get_file_size(const std::string& filename);
     static void remove_directory(const std::string& dir);