header resource line by line (#2663)

* Process one line at a time to support MSVC.

* Fix undefined variable.

* Style.

* Split header into 64k chunks on windows and re-assemble in codegen.

* style

src/ngraph/codegen/compiler.cpp

@@ -573,11 +573,27 @@ void codegen::CompilerCore::load_headers_from_resource()
     const std::string builtin_root = "";
     HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
     PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
-    // for (const std::string& search_path : builtin_search_paths)
-    // {
-    //     std::string builtin = builtin_root + search_path;
-    //     hso.AddPath(builtin, clang::frontend::System, false, false);
-    // }
+// for (const std::string& search_path : builtin_search_paths)
+// {
+//     std::string builtin = builtin_root + search_path;
+//     hso.AddPath(builtin, clang::frontend::System, false, false);
+// }
+#ifdef _WIN32
+    for (const pair<std::string, vector<std::string>>& header_info : builtin_headers)
+    {
+        std::string absolute_path = header_info.first;
+        std::string builtin = builtin_root + absolute_path;
+        std::string header_content;
+        for (const std::string& line : header_info.second)
+        {
+            header_content += line;
+        }
+        m_header_strings.emplace_back(header_content);
+        std::unique_ptr<llvm::MemoryBuffer> mb(
+            llvm::MemoryBuffer::getMemBuffer(m_header_strings.back(), builtin));
+        preprocessor_options.addRemappedFile(builtin, mb.release());
+    }
+#else
     for (const pair<std::string, std::string>& header_info : builtin_headers)
     {
         std::string absolute_path = header_info.first;
@@ -586,6 +602,7 @@ void codegen::CompilerCore::load_headers_from_resource()
             llvm::MemoryBuffer::getMemBuffer(header_info.second, builtin));
         preprocessor_options.addRemappedFile(builtin, mb.release());
     }
+#endif
 }
 
 void codegen::CompilerCore::set_precompiled_header_source(const std::string& source)

src/ngraph/codegen/compiler.hpp

@@ -93,6 +93,9 @@ private:
     std::string m_source_name;
     std::vector<std::string> m_extra_search_path_list;
     std::string m_precompiled_header_source;
+#ifdef _WIN32
+    std::vector<std::string> m_header_strings;
+#endif
 
     bool is_version_number(const std::string& path);
     int full_version_number(const std::string& path, const std::string& gpp_ver);

src/resource/header_rewrite.cpp

@@ -29,11 +29,7 @@ using namespace std;
 // Eigen has a lot of .. in their header files.
 const string rewrite_header(const string& s, const string& path)
 {
-    stringstream ss(s);
-    stringstream out;
-    for (string line; ss; getline(ss, line))
-    {
-        string original_line = line;
+    string line = s;
     // only interested in lines starging with '#include' so 8 chars minimum
     if (line.size() > 8)
     {
@@ -94,14 +90,10 @@ const string rewrite_header(const string& s, const string& path)
                         {
                             line = "#include \"" + contents + "\"";
                         }
-                            // cout << "line '" << original_line << "'\n";
-                            // cout << "rewrite to '" << line << "'\n\n";
-                        }
                     }
                 }
             }
         }
-        out << line << "\n";
     }
-    return out.str();
+    return line + "\n";
 }

src/resource/main.cpp

@@ -162,7 +162,7 @@ int main(int argc, char** argv)
     {
         size_t total_size = 0;
         size_t total_count = 0;
-        const string prefix = "pReFiX";
+        const string delim = "pReFiX";
         ofstream out(output_path);
         out << "#pragma clang diagnostic ignored \"-Weverything\"\n";
         out << "#include <vector>\n";
@@ -175,23 +175,61 @@ int main(int argc, char** argv)
             out << "        \"" << path.search_path << "\",\n";
         }
         out << "    };\n";
-
+#ifdef _WIN32
+        out << "    const std::vector<std::pair<std::string, std::vector<std::string>>> "
+               "builtin_headers =\n";
+#else
         out << "    const std::vector<std::pair<std::string, std::string>> builtin_headers =\n";
+#endif
         out << "    {\n";
         for (const ResourceInfo& path : include_paths)
         {
             for (const string& header_path : path.files)
             {
-                string header_data = read_file_to_string(header_path);
+                out << "        {";
+                out << "\"" << header_path << "\",\n";
                 string relative_path = header_path.substr(path.search_path.size() + 1);
-                header_data = rewrite_header(header_data, relative_path);
-                // header_data = uncomment(header_data);
-                total_size += header_data.size();
+                std::ifstream file(header_path);
+                if (file.is_open())
+                {
+                    std::string line;
+#ifdef _WIN32
+                    const int max_partial_size = 65500;
+                    out << "{\n";
+                    bool first_line = true;
+                    int partial_size = 0;
+                    out << "R\"" << delim << "(";
+                    while (getline(file, line))
+                    {
+                        line = rewrite_header(line, relative_path);
+                        // line = uncomment(line);
+                        total_size += line.size();
+                        partial_size += line.size();
+                        if (partial_size > max_partial_size)
+                        {
+                            out << ")" << delim << "\",\n";
+                            partial_size = line.size();
+                            out << "R\"" << delim << "(";
+                        }
+                        out << line;
+                    }
+                    out << ")" << delim << "\"";
+                    out << "}";
+#else
+                    out << "R\"" << delim << "(";
+                    while (getline(file, line))
+                    {
+                        line = rewrite_header(line, relative_path);
+                        // line = uncomment(line);
+                        total_size += line.size();
+                        out << line;
+                    }
+                    out << ")" << delim << "\"";
+#endif
+                    file.close();
+                }
+                out << "},\n";
                 total_count++;
-
-                out << "        {";
-                out << "\"" << header_path << "\",\nR\"" << prefix << "(" << header_data << ")"
-                    << prefix << "\"},\n";
             }
         }
         out << "    };\n";