Support for multiple precompiled header files (#1208)

Better CI performance

src/ngraph/codegen/compiler.cpp

@@ -71,8 +71,27 @@ using namespace llvm;
 using namespace std;
 using namespace ngraph;
 
-static codegen::StaticCompiler s_static_compiler;
-static std::mutex m_mutex;
+class CompilerInfo
+{
+public:
+    string pch_file;
+    shared_ptr<codegen::CompilerCore> compiler;
+};
+
+static unordered_map<string, CompilerInfo> s_compiler_info;
+
+static class StaticHandler
+{
+public:
+    StaticHandler() {}
+    ~StaticHandler()
+    {
+        for (const auto& p : s_compiler_info)
+        {
+            file_util::remove_file(p.second.pch_file);
+        }
+    }
+} s_static_init;
 
 codegen::Module::Module(std::unique_ptr<llvm::Module> module)
     : m_module(move(module))
@@ -89,27 +108,41 @@ std::unique_ptr<llvm::Module> codegen::Module::take_module()
 }
 
 codegen::Compiler::Compiler()
+    : m_compiler_core{}
 {
 }
 
 codegen::Compiler::~Compiler()
 {
+    m_compiler_action = nullptr;
+    m_compiler_core = nullptr;
 }
 
 void codegen::Compiler::set_precompiled_header_source(const std::string& source)
 {
-    s_static_compiler.set_precompiled_header_source(source);
+    m_precompiled_header_source = source;
 }
 
 void codegen::Compiler::add_header_search_path(const std::string& path)
 {
-    s_static_compiler.add_header_search_path(path);
+    m_header_search_paths.push_back(path);
 }
 
 std::unique_ptr<codegen::Module> codegen::Compiler::compile(const std::string& source)
 {
-    lock_guard<mutex> lock(m_mutex);
-    return s_static_compiler.compile(m_compiler_action, source);
+    // lock_guard<mutex> lock(m_mutex);
+    CompilerInfo& compiler_info = s_compiler_info[m_precompiled_header_source];
+    if (!compiler_info.compiler)
+    {
+        compiler_info.compiler = make_shared<CompilerCore>();
+        for (const string& path : m_header_search_paths)
+        {
+            compiler_info.compiler->add_header_search_path(path);
+        }
+        compiler_info.compiler->set_precompiled_header_source(m_precompiled_header_source);
+    }
+    auto rc = compiler_info.compiler->compile(m_compiler_action, source);
+    return rc;
 }
 
 static std::string GetExecutablePath(const char* Argv0)
@@ -120,9 +153,8 @@ static std::string GetExecutablePath(const char* Argv0)
     return llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
 }
 
-codegen::StaticCompiler::StaticCompiler()
-    : m_precompiled_header_valid(false)
-    , m_debuginfo_enabled((std::getenv("NGRAPH_COMPILER_DEBUGINFO_ENABLE") != nullptr))
+codegen::CompilerCore::CompilerCore()
+    : m_debuginfo_enabled((std::getenv("NGRAPH_COMPILER_DEBUGINFO_ENABLE") != nullptr))
     , m_enable_diag_output((std::getenv("NGRAPH_COMPILER_DIAG_ENABLE") != nullptr))
     , m_enable_pass_report((std::getenv("NGRAPH_COMPILER_REPORT_ENABLE") != nullptr))
     , m_source_name("code.cpp")
@@ -130,7 +162,7 @@ codegen::StaticCompiler::StaticCompiler()
     initialize();
 }
 
-void codegen::StaticCompiler::initialize()
+void codegen::CompilerCore::initialize()
 {
     m_extra_search_path_list.clear();
 
@@ -226,23 +258,23 @@ void codegen::StaticCompiler::initialize()
     diag_buffer->FlushDiagnostics(m_compiler->getDiagnostics());
 }
 
-codegen::StaticCompiler::~StaticCompiler()
+codegen::CompilerCore::~CompilerCore()
 {
     // This is causing a segfault after program terminates
     // will address later
-    // if (m_compiler)
-    // {
-    //     PreprocessorOptions& preprocessor_options =
-    //         m_compiler->getInvocation().getPreprocessorOpts();
-    //     for (auto& x : preprocessor_options.RemappedFileBuffers)
-    //     {
-    //         delete x.second;
-    //     }
-    //     m_compiler = nullptr;
-    // }
+    if (m_compiler)
+    {
+        // PreprocessorOptions& preprocessor_options =
+        //     m_compiler->getInvocation().getPreprocessorOpts();
+        // for (auto& x : preprocessor_options.RemappedFileBuffers)
+        // {
+        //     delete x.second;
+        // }
+        m_compiler = nullptr;
+    }
 }
 
-bool codegen::StaticCompiler::is_version_number(const string& path)
+bool codegen::CompilerCore::is_version_number(const string& path)
 {
     bool rc = true;
     vector<string> tokens = split(path, '.');
@@ -259,7 +291,7 @@ bool codegen::StaticCompiler::is_version_number(const string& path)
     return rc;
 }
 
-void codegen::StaticCompiler::add_header_search_path(const string& p)
+void codegen::CompilerCore::add_header_search_path(const string& p)
 {
     vector<string> paths = split(p, ';');
     for (const string& path : paths)
@@ -274,21 +306,22 @@ void codegen::StaticCompiler::add_header_search_path(const string& p)
 }
 
 std::unique_ptr<codegen::Module>
-    codegen::StaticCompiler::compile(std::unique_ptr<clang::CodeGenAction>& m_compiler_action,
-                                     const string& source)
+    codegen::CompilerCore::compile(std::unique_ptr<clang::CodeGenAction>& m_compiler_action,
+                                   const string& source)
 {
     PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
 
     preprocessor_options.RetainRemappedFileBuffers = true;
 
-    if (!m_precompiled_header_valid && m_precomiled_header_source.empty() == false)
+    CompilerInfo& compiler_info = s_compiler_info[m_precompiled_header_source];
+    if (!m_precompiled_header_source.empty() && compiler_info.pch_file.empty())
     {
-        generate_pch(m_precomiled_header_source);
+        compiler_info.pch_file = generate_pch(m_precompiled_header_source);
     }
-    if (m_precompiled_header_valid)
+    if (!compiler_info.pch_file.empty())
     {
         // Preprocessor options
-        preprocessor_options.ImplicitPCHInclude = m_pch_path;
+        preprocessor_options.ImplicitPCHInclude = compiler_info.pch_file;
         preprocessor_options.DisablePCHValidation = 0;
     }
 
@@ -329,17 +362,17 @@ std::unique_ptr<codegen::Module>
 
     if (reinitialize)
     {
-        codegen::StaticCompiler::initialize();
+        codegen::CompilerCore::initialize();
     }
 
     return result;
 }
 
-void codegen::StaticCompiler::generate_pch(const string& source)
+string codegen::CompilerCore::generate_pch(const string& source)
 {
     PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
-    m_pch_path = file_util::tmp_filename();
-    m_compiler->getFrontendOpts().OutputFile = m_pch_path;
+    string pch_path = file_util::tmp_filename();
+    m_compiler->getFrontendOpts().OutputFile = pch_path;
 
     // Map code filename to a memoryBuffer
     StringRef source_ref(source);
@@ -348,18 +381,25 @@ void codegen::StaticCompiler::generate_pch(const string& source)
 
     // Create and execute action
     clang::GeneratePCHAction* compilerAction = new clang::GeneratePCHAction();
-    if (m_compiler->ExecuteAction(*compilerAction) == true)
+    if (m_compiler->ExecuteAction(*compilerAction) == false)
+    {
+        file_util::remove_file(pch_path);
+        pch_path = "";
+    }
+    else
     {
-        m_precompiled_header_valid = true;
+        s_compiler_info[source].pch_file = pch_path;
     }
 
     buffer.release();
     preprocessor_options.RemappedFileBuffers.pop_back();
 
     delete compilerAction;
+
+    return pch_path;
 }
 
-void codegen::StaticCompiler::configure_search_path()
+void codegen::CompilerCore::configure_search_path()
 {
 #ifdef USE_BUILTIN
     load_headers_from_resource();
@@ -418,7 +458,7 @@ void codegen::StaticCompiler::configure_search_path()
 #endif
 }
 
-void codegen::StaticCompiler::load_headers_from_resource()
+void codegen::CompilerCore::load_headers_from_resource()
 {
     const string builtin_root = "";
     HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
@@ -438,12 +478,17 @@ void codegen::StaticCompiler::load_headers_from_resource()
     }
 }
 
-void codegen::StaticCompiler::set_precompiled_header_source(const std::string& source)
+void codegen::CompilerCore::set_precompiled_header_source(const std::string& source)
+{
+    m_precompiled_header_source = source;
+}
+
+const string& codegen::CompilerCore::get_precompiled_header_source() const
 {
-    m_precomiled_header_source = source;
+    return m_precompiled_header_source;
 }
 
-string codegen::StaticCompiler::find_header_version(const string& path)
+string codegen::CompilerCore::find_header_version(const string& path)
 {
     vector<string> directories;
     string rc;

src/ngraph/codegen/compiler.hpp

@@ -26,14 +26,12 @@ namespace ngraph
     {
         class Module;
         class Compiler;
-        class StaticCompiler;
-        class HeaderCache;
+        class CompilerCore;
     }
 }
 
 namespace clang
 {
-    class HeaderSearchOptions;
     class CompilerInstance;
     class CodeGenAction;
 }
@@ -65,34 +63,36 @@ public:
     std::unique_ptr<clang::CodeGenAction>& get_compiler_action() { return m_compiler_action; }
 private:
     std::unique_ptr<clang::CodeGenAction> m_compiler_action;
+    std::shared_ptr<CompilerCore> m_compiler_core;
+    std::string m_precompiled_header_source;
+    std::vector<std::string> m_header_search_paths;
 };
 
-class ngraph::codegen::StaticCompiler
+class ngraph::codegen::CompilerCore
 {
 public:
-    StaticCompiler();
-    ~StaticCompiler();
+    CompilerCore();
+    ~CompilerCore();
 
     void set_debuginfo_enabled(bool state) { m_debuginfo_enabled = state; }
     bool is_debuginfo_enabled() { return m_debuginfo_enabled; }
     void set_precompiled_header_source(const std::string& source);
+    const std::string& get_precompiled_header_source() const;
     void add_header_search_path(const std::string& path);
 
     std::unique_ptr<ngraph::codegen::Module>
         compile(std::unique_ptr<clang::CodeGenAction>& compiler_action, const std::string& source);
-    void generate_pch(const std::string& source);
+    std::string generate_pch(const std::string& source);
     void initialize();
 
 private:
     std::unique_ptr<clang::CompilerInstance> m_compiler;
-    bool m_precompiled_header_valid;
     bool m_debuginfo_enabled;
     bool m_enable_diag_output;
     bool m_enable_pass_report;
     std::string m_source_name;
     std::vector<std::string> m_extra_search_path_list;
-    std::string m_pch_path;
-    std::string m_precomiled_header_source;
+    std::string m_precompiled_header_source;
 
     bool is_version_number(const std::string& path);
     std::string find_header_version(const std::string& path);