header resource line by line (#2663)

* Process one line at a time to support MSVC.

* Fix undefined variable.

* Style.

* Split header into 64k chunks on windows and re-assemble in codegen.

* style

src/ngraph/codegen/compiler.cpp

@@ -573,11 +573,27 @@ void codegen::CompilerCore::load_headers_from_resource()
     const std::string builtin_root = "";
     HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
     PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
-    // for (const std::string& search_path : builtin_search_paths)
-    // {
-    //     std::string builtin = builtin_root + search_path;
-    //     hso.AddPath(builtin, clang::frontend::System, false, false);
-    // }
+// for (const std::string& search_path : builtin_search_paths)
+// {
+//     std::string builtin = builtin_root + search_path;
+//     hso.AddPath(builtin, clang::frontend::System, false, false);
+// }
+#ifdef _WIN32
+    for (const pair<std::string, vector<std::string>>& header_info : builtin_headers)
+    {
+        std::string absolute_path = header_info.first;
+        std::string builtin = builtin_root + absolute_path;
+        std::string header_content;
+        for (const std::string& line : header_info.second)
+        {
+            header_content += line;
+        }
+        m_header_strings.emplace_back(header_content);
+        std::unique_ptr<llvm::MemoryBuffer> mb(
+            llvm::MemoryBuffer::getMemBuffer(m_header_strings.back(), builtin));
+        preprocessor_options.addRemappedFile(builtin, mb.release());
+    }
+#else
     for (const pair<std::string, std::string>& header_info : builtin_headers)
     {
         std::string absolute_path = header_info.first;
@@ -586,6 +602,7 @@ void codegen::CompilerCore::load_headers_from_resource()
             llvm::MemoryBuffer::getMemBuffer(header_info.second, builtin));
         preprocessor_options.addRemappedFile(builtin, mb.release());
     }
+#endif
 }
 
 void codegen::CompilerCore::set_precompiled_header_source(const std::string& source)

src/ngraph/codegen/compiler.hpp

@@ -93,6 +93,9 @@ private:
     std::string m_source_name;
     std::vector<std::string> m_extra_search_path_list;
     std::string m_precompiled_header_source;
+#ifdef _WIN32
+    std::vector<std::string> m_header_strings;
+#endif
 
     bool is_version_number(const std::string& path);
     int full_version_number(const std::string& path, const std::string& gpp_ver);

src/resource/header_rewrite.cpp

@@ -29,79 +29,71 @@ using namespace std;
 // Eigen has a lot of .. in their header files.
 const string rewrite_header(const string& s, const string& path)
 {
-    stringstream ss(s);
-    stringstream out;
-    for (string line; ss; getline(ss, line))
+    string line = s;
+    // only interested in lines starging with '#include' so 8 chars minimum
+    if (line.size() > 8)
     {
-        string original_line = line;
-        // only interested in lines starging with '#include' so 8 chars minimum
-        if (line.size() > 8)
+        // skip whitespace
+        size_t pos = line.find_first_not_of(" \t");
+        if (pos != string::npos && line[pos] == '#' && pos < line.size() - 7)
         {
-            // skip whitespace
-            size_t pos = line.find_first_not_of(" \t");
-            if (pos != string::npos && line[pos] == '#' && pos < line.size() - 7)
+            string directive = line;
+            pos = directive.find_first_not_of(" \t", pos + 1);
+            if (pos != string::npos)
             {
-                string directive = line;
-                pos = directive.find_first_not_of(" \t", pos + 1);
-                if (pos != string::npos)
-                {
-                    directive = directive.substr(pos);
-                }
-                pos = directive.find_first_of(" \t", pos + 1);
-                directive = directive.substr(0, pos);
-                if (directive == "include")
+                directive = directive.substr(pos);
+            }
+            pos = directive.find_first_of(" \t", pos + 1);
+            directive = directive.substr(0, pos);
+            if (directive == "include")
+            {
+                auto line_offset = line.find_first_of("\"<");
+                if (line_offset != string::npos)
                 {
-                    auto line_offset = line.find_first_of("\"<");
-                    if (line_offset != string::npos)
+                    string include = line.substr(line_offset);
+                    string contents = include.substr(1, include.size() - 2);
+                    if (include[1] == '.')
                     {
-                        string include = line.substr(line_offset);
-                        string contents = include.substr(1, include.size() - 2);
-                        if (include[1] == '.')
+                        if (include[2] == '/')
                         {
-                            if (include[2] == '/')
-                            {
-                                // include starts with './'
-                                // rewrite "./blah.h" to "blah.h"
-                                contents = contents.substr(2);
-                            }
-                            else
-                            {
-                                // include starts with '../'
-                                // count number of '../' in string
-                                size_t offset = 0;
-                                size_t depth = 0;
-                                while (contents.substr(offset, 3) == "../")
-                                {
-                                    depth++;
-                                    offset += 3;
-                                }
-                                string trimmed = contents.substr(offset);
-                                vector<string> parts = split(path, '/');
-                                parts.pop_back();
-                                size_t result_depth = parts.size() - depth;
-                                string added_path;
-                                for (size_t i = 0; i < result_depth; i++)
-                                {
-                                    added_path += parts[i] + "/";
-                                }
-                                contents = added_path + trimmed;
-                            }
-                            if (include[0] == '<')
+                            // include starts with './'
+                            // rewrite "./blah.h" to "blah.h"
+                            contents = contents.substr(2);
+                        }
+                        else
+                        {
+                            // include starts with '../'
+                            // count number of '../' in string
+                            size_t offset = 0;
+                            size_t depth = 0;
+                            while (contents.substr(offset, 3) == "../")
                             {
-                                line = "#include <" + contents + ">";
+                                depth++;
+                                offset += 3;
                             }
-                            else
+                            string trimmed = contents.substr(offset);
+                            vector<string> parts = split(path, '/');
+                            parts.pop_back();
+                            size_t result_depth = parts.size() - depth;
+                            string added_path;
+                            for (size_t i = 0; i < result_depth; i++)
                             {
-                                line = "#include \"" + contents + "\"";
+                                added_path += parts[i] + "/";
                             }
-                            // cout << "line '" << original_line << "'\n";
-                            // cout << "rewrite to '" << line << "'\n\n";
+                            contents = added_path + trimmed;
+                        }
+                        if (include[0] == '<')
+                        {
+                            line = "#include <" + contents + ">";
+                        }
+                        else
+                        {
+                            line = "#include \"" + contents + "\"";
                         }
                     }
                 }
             }
         }
-        out << line << "\n";
     }
-    return out.str();
+    return line + "\n";
 }

src/resource/main.cpp

@@ -162,7 +162,7 @@ int main(int argc, char** argv)
     {
         size_t total_size = 0;
         size_t total_count = 0;
-        const string prefix = "pReFiX";
+        const string delim = "pReFiX";
         ofstream out(output_path);
         out << "#pragma clang diagnostic ignored \"-Weverything\"\n";
         out << "#include <vector>\n";
@@ -175,23 +175,61 @@ int main(int argc, char** argv)
             out << "        \"" << path.search_path << "\",\n";
         }
         out << "    };\n";
-
+#ifdef _WIN32
+        out << "    const std::vector<std::pair<std::string, std::vector<std::string>>> "
+               "builtin_headers =\n";
+#else
         out << "    const std::vector<std::pair<std::string, std::string>> builtin_headers =\n";
+#endif
         out << "    {\n";
         for (const ResourceInfo& path : include_paths)
         {
             for (const string& header_path : path.files)
             {
-                string header_data = read_file_to_string(header_path);
+                out << "        {";
+                out << "\"" << header_path << "\",\n";
                 string relative_path = header_path.substr(path.search_path.size() + 1);
-                header_data = rewrite_header(header_data, relative_path);
-                // header_data = uncomment(header_data);
-                total_size += header_data.size();
+                std::ifstream file(header_path);
+                if (file.is_open())
+                {
+                    std::string line;
+#ifdef _WIN32
+                    const int max_partial_size = 65500;
+                    out << "{\n";
+                    bool first_line = true;
+                    int partial_size = 0;
+                    out << "R\"" << delim << "(";
+                    while (getline(file, line))
+                    {
+                        line = rewrite_header(line, relative_path);
+                        // line = uncomment(line);
+                        total_size += line.size();
+                        partial_size += line.size();
+                        if (partial_size > max_partial_size)
+                        {
+                            out << ")" << delim << "\",\n";
+                            partial_size = line.size();
+                            out << "R\"" << delim << "(";
+                        }
+                        out << line;
+                    }
+                    out << ")" << delim << "\"";
+                    out << "}";
+#else
+                    out << "R\"" << delim << "(";
+                    while (getline(file, line))
+                    {
+                        line = rewrite_header(line, relative_path);
+                        // line = uncomment(line);
+                        total_size += line.size();
+                        out << line;
+                    }
+                    out << ")" << delim << "\"";
+#endif
+                    file.close();
+                }
+                out << "},\n";
                 total_count++;
-
-                out << "        {";
-                out << "\"" << header_path << "\",\nR\"" << prefix << "(" << header_data << ")"
-                    << prefix << "\"},\n";
             }
         }
         out << "    };\n";