Skip to content

N
ngraph
Commit bfb48511 authored by Sang Ik Lee's avatar Sang Ik Lee Committed by Robert Kimball
Browse files
Options

Improve codegen's header search path finding logic for various Linux distributions. (#2468)

parent 8665e27e
Hide whitespace changes
Inline Side-by-side
Showing with 224 additions and 70 deletions
src/ngraph/codegen/CMakeLists.txt
View file @ bfb48511
......@@ -33,10 +33,10 @@ add_library(codegen SHARED ${SRC})
# This must be kept in sync with the LLVM + Clang version in use
set_source_files_properties(compiler.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti")
get_target_property(LLVM_LIB_DIR libllvm INTERFACE_INCLUDE_DIRECTORIES)
get_target_property(LLVM_INCLUDE_DIR libllvm INTERFACE_INCLUDE_DIRECTORIES)
# find_file(HEADER_1 cmath HINTS /usr/include/c++/7)
get_filename_component(LLVM_LIB_DIR ${LLVM_LIB_DIR}/../lib/clang/5.0.2/include ABSOLUTE)
get_filename_component(CLANG_INCLUDE_DIR ${LLVM_INCLUDE_DIR}/../lib/clang/5.0.2/include ABSOLUTE)
if(NGRAPH_CPU_ENABLE)
get_target_property(MKLDNN_INCLUDE_DIR libmkldnn INTERFACE_INCLUDE_DIRECTORIES)
......@@ -45,7 +45,7 @@ if(NGRAPH_CPU_ENABLE)
list(APPEND HEADER_SEARCH_DEFINES MKLDNN_HEADERS_PATH="${MKLDNN_INCLUDE_DIR}")
endif()
list(APPEND HEADER_SEARCH_DEFINES CLANG_BUILTIN_HEADERS_PATH="${LLVM_LIB_DIR}")
list(APPEND HEADER_SEARCH_DEFINES CLANG_BUILTIN_HEADERS_PATH="${CLANG_INCLUDE_DIR}")
list(APPEND HEADER_SEARCH_DEFINES NGRAPH_HEADERS_PATH="${NGRAPH_INCLUDE_PATH}")
if(NGRAPH_DISTRIBUTED_ENABLE)
......@@ -76,7 +76,7 @@ set_source_files_properties(compiler.cpp PROPERTIES COMPILE_DEFINITIONS "${HEADE
# Generate the resource file containing all headers used by the codegen compiler
add_custom_target(header_resource
resource_generator --output ${CMAKE_BINARY_DIR}/header_resource.hpp --base codegen
DEPENDS resource_generator ext_llvm
DEPENDS resource_generator
BYPRODUCTS
)
if (NGRAPH_CPU_ENABLE)
......@@ -99,5 +99,5 @@ if (NGRAPH_CPU_ENABLE)
endif()
target_include_directories(codegen SYSTEM PRIVATE ${CMAKE_BINARY_DIR})
target_link_libraries(codegen PRIVATE libllvm ngraph)
set_target_properties(codegen PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR})
install(TARGETS codegen DESTINATION ${NGRAPH_INSTALL_LIB})
src/ngraph/codegen/compiler.cpp
View file @ bfb48511
......@@ -15,6 +15,7 @@
//*****************************************************************************
#include <iostream>
#include <string>
#include <clang/Basic/DiagnosticOptions.h>
#include <clang/Basic/TargetInfo.h>
......@@ -75,11 +76,11 @@ using namespace ngraph;
class CompilerInfo
{
public:
string pch_file;
std::string pch_file;
shared_ptr<codegen::CompilerCore> compiler;
};
static unordered_map<string, CompilerInfo> s_compiler_info;
static unordered_map<std::string, CompilerInfo> s_compiler_info;
static class StaticHandler
{
......@@ -136,7 +137,7 @@ std::unique_ptr<codegen::Module> codegen::Compiler::compile(const std::string& s
if (!compiler_info.compiler)
{
compiler_info.compiler = make_shared<CompilerCore>();
for (const string& path : m_header_search_paths)
for (const std::string& path : m_header_search_paths)
{
compiler_info.compiler->add_header_search_path(path);
}
......@@ -238,7 +239,7 @@ void codegen::CompilerCore::initialize()
// CodeGen options
auto& CGO = m_compiler->getInvocation().getCodeGenOpts();
CGO.OptimizationLevel = 3;
CGO.RelocationModel = "static";
CGO.RelocationModel = llvm::Reloc::Model::Static;
// CGO.CodeModel = "medium";
CGO.ThreadModel = "posix";
CGO.FloatABI = "hard";
......@@ -276,11 +277,11 @@ codegen::CompilerCore::~CompilerCore()
}
}
bool codegen::CompilerCore::is_version_number(const string& path)
bool codegen::CompilerCore::is_version_number(const std::string& path)
{
bool rc = true;
vector<string> tokens = split(path, '.');
for (string s : tokens)
vector<std::string> tokens = split(path, '.');
for (std::string s : tokens)
{
for (char c : s)
{
......@@ -293,24 +294,27 @@ bool codegen::CompilerCore::is_version_number(const string& path)
return rc;
}
void codegen::CompilerCore::add_header_search_path(const string& p)
void codegen::CompilerCore::add_header_search_path(const std::string& p, bool check_path)
{
vector<string> paths = split(p, ';');
for (const string& path : paths)
vector<std::string> paths = split(p, ';');
for (const std::string& path : paths)
{
if (find(m_extra_search_path_list.begin(), m_extra_search_path_list.end(), path) ==
m_extra_search_path_list.end())
{
m_extra_search_path_list.push_back(path);
HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
hso.AddPath(path, clang::frontend::System, false, false);
if (!check_path || file_util::exists(path))
{
m_extra_search_path_list.push_back(path);
HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
hso.AddPath(path, clang::frontend::System, false, false);
}
}
}
}
std::unique_ptr<codegen::Module>
codegen::CompilerCore::compile(std::unique_ptr<clang::CodeGenAction>& m_compiler_action,
const string& source)
const std::string& source)
{
PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
......@@ -371,10 +375,10 @@ std::unique_ptr<codegen::Module>
return result;
}
string codegen::CompilerCore::generate_pch(const string& source)
std::string codegen::CompilerCore::generate_pch(const std::string& source)
{
PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
string pch_path = file_util::tmp_filename();
std::string pch_path = file_util::tmp_filename();
m_compiler->getFrontendOpts().OutputFile = pch_path;
// Map code filename to a memoryBuffer
......@@ -408,7 +412,6 @@ void codegen::CompilerCore::configure_search_path()
load_headers_from_resource();
#endif
#if defined(__APPLE__)
#ifdef EIGEN_HEADERS_PATH
add_header_search_path(EIGEN_HEADERS_PATH);
#endif
......@@ -419,46 +422,128 @@ void codegen::CompilerCore::configure_search_path()
add_header_search_path(TBB_HEADERS_PATH);
#endif
add_header_search_path(NGRAPH_HEADERS_PATH);
add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
#if defined(__APPLE__)
// Is it OK to use AppleClang c++ headers for Clang?
add_header_search_path("/Library/Developer/CommandLineTools/usr/include/c++/v1");
add_header_search_path("/usr/local/include");
add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
std::string mojave_isysroot = file_util::path_join(
"/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs",
"MacOSX10.14.sdk");
if (file_util::exists(mojave_isysroot))
{
add_header_search_path(file_util::path_join(mojave_isysroot, "usr/include"));
}
else
{
add_header_search_path("/usr/include");
}
#else
// Add base toolchain-supplied header paths
// Ideally one would use the Linux toolchain definition in clang/lib/Driver/ToolChains.h
// But that's a private header and isn't part of the public libclang API
// Instead of re-implementing all of that functionality in a custom toolchain
// just hardcode the paths relevant to frequently used build/test machines for now
add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
#ifdef EIGEN_HEADERS_PATH
add_header_search_path(EIGEN_HEADERS_PATH);
#endif
#ifdef MKLDNN_HEADERS_PATH
add_header_search_path(MKLDNN_HEADERS_PATH);
#endif
#ifdef TBB_HEADERS_PATH
add_header_search_path(TBB_HEADERS_PATH);
#endif
add_header_search_path(NGRAPH_HEADERS_PATH);
string header_version = find_header_version("/usr/include/c++");
string os_specific_path =
find_os_specific_path(file_util::path_join("/usr/include/c++", header_version));
//
// Redhat/CentOS g++ location
// /usr/bin/g++
// Redhat/CentOS g++ default include path
// /usr/include/c++/4.8.5
// /usr/include/c++/4.8.5/x86_64-redhat-linux
// /usr/include/c++/4.8.5/backward
// /usr/lib/gcc/x86_64-redhat-linux/4.8.5/include
// /usr/local/include
// /usr/include
// Redhat/CentOS clang++ default include path
// /usr/include/c++/4.8.5
// /usr/include/c++/4.8.5/x86_64-redhat-linux
// /usr/include/c++/4.8.5/backward
// /usr/local/include
// /usr/lib/clang/3.4.2/include
// /usr/include
// Common path
// /usr/include/c++/4.8.5
// /usr/include/c++/4.8.5/x86_64-redhat-linux
// /usr/include/c++/4.8.5/backward
// /usr/local/include
// /usr/include
//
// Ubuntu g++ location
// /usr/bin/g++-<VER>
// Ubuntu g++ default include path
// /usr/include/c++/5
// /usr/include/x86_64-linux-gnu/c++/5
// /usr/include/c++/5/backward
// /usr/lib/gcc/x86_64-linux-gnu/5/include
// /usr/local/include
// /usr/lib/gcc/x86_64-linux-gnu/5/include-fixed
// /usr/include/x86_64-linux-gnu
// /usr/include
// Ubuntu clang++ default include path
// /usr/include/c++/5.4.0
// /usr/include/x86_64-linux-gnu/c++/5.4.0
// /usr/include/c++/5.4.0/backward
// /usr/local/include
// <path to clang>/lib/clang/7.0.0/include
// /usr/include/x86_64-linux-gnu
// /usr/include
// Common path
// /usr/include/c++/5.4.0
// /usr/include/x86_64-linux-gnu/c++/5.4.0
// /usr/include/c++/5.4.0/backward
// /usr/local/include
// /usr/include/x86_64-linux-gnu
// /usr/include
//
// Overall steps
// 1. Check if system has devtoolset and set header path prefix
// 2. For highest gcc version, find the highest header version
// 3. Set Ubuntu/Redhat common path, set platform specific paths if they exist
std::string toolpath = find_rh_devtoolset_path();
bool has_toolpath = !toolpath.empty();
std::string usr_prefix = has_toolpath ? file_util::path_join(toolpath, "usr") : "/usr";
std::string cpp_header_prefix = file_util::path_join(usr_prefix, "include/c++");
// Only Ubuntu:
// Find highest g++ version (8, 7, 6, 5, 4.9, 4.8)
// Common:
// For version X (if has version), find the highest X.Y.Z for header version
std::string header_version = find_header_version(usr_prefix);
std::string cpp_header_root = file_util::path_join(cpp_header_prefix, header_version);
std::string os_specific_path = find_os_specific_path(cpp_header_root);
// Common
add_header_search_path(cpp_header_root);
// Ubuntu only
add_header_search_path(
file_util::path_join("/usr/include/x86_64-linux-gnu/c++/", header_version), true);
// /usr/include/c++/7
add_header_search_path(file_util::path_join("/usr/include/c++/", header_version));
// Redhat/CentOS only
add_header_search_path(file_util::path_join(cpp_header_root, os_specific_path), true);
// /usr/include/x86_64-linux-gnu/c++/7
add_header_search_path(
file_util::path_join("/usr/include/x86_64-linux-gnu/c++/", header_version));
// Common
add_header_search_path(file_util::path_join(cpp_header_root, "backward"));
add_header_search_path(
file_util::path_join("/usr/lib/gcc/x86_64-linux-gnu/", header_version, "/include"));
// Common
add_header_search_path("/usr/local/include");
add_header_search_path(
file_util::path_join("/usr/include/c++/", header_version, os_specific_path));
add_header_search_path(
file_util::path_join("/usr/lib/gcc/x86_64-linux-gnu/", header_version, "/include-fixed"));
add_header_search_path("/usr/include/x86_64-linux-gnu");
// From Clang
add_header_search_path(CLANG_BUILTIN_HEADERS_PATH);
// Ubuntu only
add_header_search_path("/usr/include/x86_64-linux-gnu", true);
// Redhat/CentOS only
if (has_toolpath)
{
add_header_search_path(file_util::path_join(cpp_header_prefix, "usr/include"));
}
// Common
add_header_search_path("/usr/include");
#endif
......@@ -485,18 +570,18 @@ void codegen::CompilerCore::configure_search_path()
void codegen::CompilerCore::load_headers_from_resource()
{
const string builtin_root = "";
const std::string builtin_root = "";
HeaderSearchOptions& hso = m_compiler->getInvocation().getHeaderSearchOpts();
PreprocessorOptions& preprocessor_options = m_compiler->getInvocation().getPreprocessorOpts();
// for (const string& search_path : builtin_search_paths)
// for (const std::string& search_path : builtin_search_paths)
// {
// string builtin = builtin_root + search_path;
// std::string builtin = builtin_root + search_path;
// hso.AddPath(builtin, clang::frontend::System, false, false);
// }
for (const pair<string, string>& header_info : builtin_headers)
for (const pair<std::string, std::string>& header_info : builtin_headers)
{
string absolute_path = header_info.first;
string builtin = builtin_root + absolute_path;
std::string absolute_path = header_info.first;
std::string builtin = builtin_root + absolute_path;
std::unique_ptr<llvm::MemoryBuffer> mb(
llvm::MemoryBuffer::getMemBuffer(header_info.second, builtin));
preprocessor_options.addRemappedFile(builtin, mb.release());
......@@ -508,43 +593,91 @@ void codegen::CompilerCore::set_precompiled_header_source(const std::string& sou
m_precompiled_header_source = source;
}
const string& codegen::CompilerCore::get_precompiled_header_source() const
const std::string& codegen::CompilerCore::get_precompiled_header_source() const
{
return m_precompiled_header_source;
}
string codegen::CompilerCore::find_header_version(const string& path)
int codegen::CompilerCore::full_version_number(const std::string& path, const std::string& gpp_ver)
{
vector<string> directories;
string rc;
// check if header version is compatible with g++ version
vector<std::string> tokens = split(path, '.');
if (!gpp_ver.empty())
{
vector<std::string> gpp_tokens = split(gpp_ver, '.');
for (int i = 0; i < gpp_tokens.size(); i++)
{
if (gpp_tokens[i].compare(tokens[i]) != 0)
{
return 0;
}
}
}
// create full version number and return
std::string full_version = {};
// Assume versioning like X.Y.Z
int padding = 3 - tokens.size();
for (std::string s : tokens)
{
full_version.append(s);
}
for (int i = 0; i < padding; i++)
{
full_version.append("0");
}
return std::stoi(full_version);
}
std::string codegen::CompilerCore::find_header_version(const std::string& path)
{
// Step 1: find highest g++ version
std::string gpp_prefix = file_util::path_join(path, "bin/g++-");
std::string gpp_ver = {};
for (auto i : {"8", "7", "6", "5", "4.9", "4.8"})
{
if (file_util::exists(gpp_prefix + i))
{
gpp_ver = i;
break;
}
}
// Step 2: find highest version of header file that matches g++ version
vector<std::string> directories;
std::string rc;
auto f = [&](const std::string& file, bool is_dir) {
if (is_dir)
{
directories.push_back(file);
}
};
file_util::iterate_files(path, f);
for (const string& dir : directories)
file_util::iterate_files(file_util::path_join(path, "include/c++"), f);
int full_version = 0;
for (const std::string& dir : directories)
{
string dir_name = file_util::get_file_name(dir);
std::string dir_name = file_util::get_file_name(dir);
if (is_version_number(dir_name))
{
rc = dir_name;
break;
int tmp_version = full_version_number(dir_name, gpp_ver);
if (tmp_version > full_version)
{
full_version = tmp_version;
rc = dir_name;
}
}
}
return rc;
}
string codegen::CompilerCore::find_os_specific_path(const string& path)
std::string codegen::CompilerCore::find_os_specific_path(const std::string& path)
{
string rc;
std::string rc;
auto f = [&](const std::string& file, bool is_dir) {
if (is_dir)
{
const string prefix = "x86_64-";
const string suffix = "-linux";
string path = file_util::get_file_name(file);
const std::string prefix = "x86_64-";
const std::string suffix = "-linux";
std::string path = file_util::get_file_name(file);
if (path.size() > (prefix.size() + suffix.size()) &&
path.compare(0, prefix.size(), prefix) == 0 &&
path.compare(path.size() - suffix.size(), suffix.size(), suffix) == 0)
......@@ -557,3 +690,22 @@ string codegen::CompilerCore::find_os_specific_path(const string& path)
file_util::iterate_files(path, f);
return rc;
}
std::string codegen::CompilerCore::find_rh_devtoolset_path()
{
// Redhat/CentOS has devtoolset level support
// Support 8, 7, 6, 5, 4, 3, 2
// Find highest toolset version X
// Find highest header version X.Y.Z
std::string toolsetprefix("/opt/rh/devtoolset-");
for (auto i : {"8", "7", "6", "5", "4", "3", "2"})
{
std::string toolpath = file_util::path_join(toolsetprefix + i, "root");
if (file_util::exists(toolpath))
{
return toolpath;
}
}
return {};
}
src/ngraph/codegen/compiler.hpp
View file @ bfb48511
......@@ -78,7 +78,7 @@ public:
bool is_debuginfo_enabled() { return m_debuginfo_enabled; }
void set_precompiled_header_source(const std::string& source);
const std::string& get_precompiled_header_source() const;
void add_header_search_path(const std::string& path);
void add_header_search_path(const std::string& path, bool check_path = false);
std::unique_ptr<ngraph::codegen::Module>
compile(std::unique_ptr<clang::CodeGenAction>& compiler_action, const std::string& source);
......@@ -95,8 +95,10 @@ private:
std::string m_precompiled_header_source;
bool is_version_number(const std::string& path);
int full_version_number(const std::string& path, const std::string& gpp_ver);
std::string find_header_version(const std::string& path);
std::string find_os_specific_path(const std::string& path);
std::string find_rh_devtoolset_path();
void configure_search_path();
void load_headers_from_resource();
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment