[ONNXIFI] implement onnxGetBackendIDs() interface function (#1546)

* onnx: add missing header files
Signed-off-by: Artur Wojcik <artur.wojcik@intel.com>

* onnxifi: implementation of onnxGetBackendIDs
Signed-off-by: Artur Wojcik <artur.wojcik@intel.com>

* onnxifi: add unit tests for onnxGetBackendIDs
Signed-off-by: Artur Wojcik <artur.wojcik@intel.com>

* onnxifi: change std::out_of_range to std::length_error
Signed-off-by: Artur Wojcik <artur.wojcik@intel.com>

* onnxifi: after review changes
Signed-off-by: Artur Wojcik <artur.wojcik@intel.com>

src/ngraph/frontend/onnx_import/CMakeLists.txt

@@ -68,6 +68,7 @@ add_library(onnx_import STATIC
         op/unsqueeze.cpp
         op/unsqueeze.hpp
         ops_bridge.cpp
+        ops_bridge.hpp
         utils/broadcasting.cpp
         utils/broadcasting.hpp
         utils/convpool.cpp

src/ngraph/frontend/onnxifi/CMakeLists.txt

@@ -14,7 +14,7 @@
 # limitations under the License.
 # ******************************************************************************
 
-add_library(onnxifi-ngraph SHARED onnxifi.cpp)
+add_library(onnxifi-ngraph SHARED onnxifi.cpp backend.hpp backend_manager.hpp backend_manager.cpp)
 target_link_libraries(onnxifi-ngraph PRIVATE ngraph)
 
 add_dependencies(onnxifi-ngraph onnx::libonnx)

src/ngraph/frontend/onnxifi/backend.hpp

+//*****************************************************************************
+// Copyright 2017-2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include <memory>  // std::shared_ptr
+#include <string>  // std::string
+#include <utility> // std::move
+#include <vector>  // std::vector
+
+#include "ngraph/function.hpp"
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace onnxifi
+    {
+        /// \brief ONNXIFI extensions to nGraph backend
+        class Backend
+        {
+        public:
+            Backend(const Backend&) = delete;
+            Backend& operator=(const Backend&) = delete;
+
+            Backend(Backend&&) = default;
+            Backend& operator=(Backend&&) = default;
+
+            Backend() = delete;
+
+            explicit Backend(const std::string& type)
+                : m_type{type}
+            {
+            }
+
+            const std::string& get_type() const { return m_type; }
+            bool compile(const std::shared_ptr<Function>& function) const
+            {
+                return get().compile(function);
+            }
+
+            bool call(const std::shared_ptr<Function>& function,
+                      const std::vector<std::shared_ptr<runtime::TensorView>>& outputs,
+                      const std::vector<std::shared_ptr<runtime::TensorView>>& inputs) const
+            {
+                return get().call(function, outputs, inputs);
+            }
+
+            bool call_with_validate(
+                const std::shared_ptr<Function>& function,
+                const std::vector<std::shared_ptr<runtime::TensorView>>& outputs,
+                const std::vector<std::shared_ptr<runtime::TensorView>>& inputs) const
+            {
+                return get().call_with_validate(function, outputs, inputs);
+            }
+
+        private:
+            std::string m_type{};
+            mutable std::shared_ptr<runtime::Backend> m_backend{nullptr};
+
+            runtime::Backend& get() const
+            {
+                if (m_backend == nullptr)
+                {
+                    m_backend = runtime::Backend::create(m_type);
+                }
+                return *m_backend;
+            }
+        };
+
+    } // namespace onnxifi
+
+} // namespace ngraph

src/ngraph/frontend/onnxifi/backend_manager.cpp

+//*****************************************************************************
+// Copyright 2017-2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include <cstdlib>   // std::size_t, std::uintptr_t
+#include <stdexcept> // std::invalid_agrument, std::out_of_rage
+
+#include <onnxifi.h>
+
+#include "ngraph/runtime/backend_manager.hpp"
+
+#include "backend.hpp"
+#include "backend_manager.hpp"
+
+namespace ngraph
+{
+    namespace onnxifi
+    {
+        BackendManager::BackendManager()
+        {
+            // Create ONNXIFI backend for each registered nGraph backend.
+            // Use pointer to temporary to capture the unique handle. The handles
+            // must be consistent within the single session.
+            // In spec, backends are hot-pluggable. This means two calls to
+            // onnxGetBackendIDs() may result in different number of backends.
+            // For now, we don't do the re-discovery.
+            auto registered_backends = runtime::BackendManager::get_registered_backends();
+            for (const auto& type : registered_backends)
+            {
+                m_registered_backends.emplace(reinterpret_cast<std::uintptr_t>(&type),
+                                              Backend{type});
+            }
+        }
+
+        void BackendManager::get_registered_ids(::onnxBackendID* backend_ids,
+                                                std::size_t* count) const
+        {
+            if (count == nullptr)
+            {
+                throw std::invalid_argument{"null pointer"};
+            }
+            std::size_t requested{*count};
+            *count = m_registered_backends.size();
+            if ((requested < *count) || (backend_ids == nullptr))
+            {
+                throw std::length_error{"not enough space"};
+            }
+            {
+                std::lock_guard<decltype(m_mutex)> lock{m_mutex};
+                std::transform(std::begin(m_registered_backends),
+                               std::end(m_registered_backends),
+                               backend_ids,
+                               [](const std::map<std::uintptr_t, Backend>::value_type& pair)
+                                   -> ::onnxBackendID {
+                                   return reinterpret_cast<::onnxBackendID>(pair.first);
+                               });
+            }
+        }
+    }
+}

src/ngraph/frontend/onnxifi/backend_manager.hpp

+//*****************************************************************************
+// Copyright 2017-2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include <cstddef> // std::size_t, std::uintptr_t
+#include <map>     // std::map
+#include <mutex>   // std::mutex
+
+#include <onnxifi.h>
+
+#include "ngraph/runtime/backend.hpp"
+
+#include "backend.hpp"
+
+namespace ngraph
+{
+    namespace onnxifi
+    {
+        /// \brief ONNXIFI backend manager
+        class BackendManager
+        {
+        public:
+            BackendManager(const BackendManager&) = delete;
+            BackendManager& operator=(const BackendManager&) = delete;
+
+            BackendManager(BackendManager&&) = delete;
+            BackendManager& operator=(BackendManager&&) = delete;
+
+            static void get_backend_ids(::onnxBackendID* backend_ids, std::size_t* count)
+            {
+                instance().get_registered_ids(backend_ids, count);
+            }
+
+            static void unregister(::onnxBackendID backend_id)
+            {
+                instance().unregister_backend(backend_id);
+            }
+
+            static const Backend& get(::onnxBackendID backend_id)
+            {
+                return instance().get_backend(backend_id);
+            }
+
+        private:
+            mutable std::mutex m_mutex{};
+            std::map<std::uintptr_t, Backend> m_registered_backends{};
+
+            BackendManager();
+
+            static BackendManager& instance()
+            {
+                static BackendManager backend_manager;
+                return backend_manager;
+            }
+
+            void unregister_backend(std::uintptr_t id)
+            {
+                std::lock_guard<decltype(m_mutex)> lock{m_mutex};
+                m_registered_backends.erase(id);
+            }
+
+            void unregister_backend(::onnxBackendID id)
+            {
+                return unregister_backend(reinterpret_cast<std::uintptr_t>(id));
+            }
+
+            void get_registered_ids(::onnxBackendID* backend_ids, std::size_t* count) const;
+
+            const Backend& get_backend(std::uintptr_t id) const
+            {
+                std::lock_guard<decltype(m_mutex)> lock{m_mutex};
+                return m_registered_backends.at(id);
+            }
+
+            const Backend& get_backend(::onnxBackendID id) const
+            {
+                return get_backend(reinterpret_cast<std::uintptr_t>(id));
+            }
+        };
+
+    } // namespace onnxifi
+
+} // namespace ngraph

src/ngraph/frontend/onnxifi/onnxifi.cpp

@@ -16,18 +16,38 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <stdexcept>
+
 #include <onnxifi.h>
 
+#include "backend_manager.hpp"
+
 extern "C" {
 
 ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
     onnxGetBackendIDs(onnxBackendID* backendIDs, std::size_t* numBackends)
 {
-    if ((backendIDs == nullptr) || (numBackends == nullptr))
+    try
+    {
+        ngraph::onnxifi::BackendManager::get_backend_ids(backendIDs, numBackends);
+        return ONNXIFI_STATUS_SUCCESS;
+    }
+    catch (const std::invalid_argument&)
     {
         return ONNXIFI_STATUS_INVALID_POINTER;
     }
-    return ONNXIFI_STATUS_INTERNAL_ERROR;
+    catch (const std::bad_alloc&)
+    {
+        return ONNXIFI_STATUS_NO_SYSTEM_MEMORY;
+    }
+    catch (const std::length_error&)
+    {
+        return ONNXIFI_STATUS_FALLBACK;
+    }
+    catch (...)
+    {
+        return ONNXIFI_STATUS_INTERNAL_ERROR;
+    }
 }
 
 ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI

test/CMakeLists.txt

@@ -48,6 +48,9 @@ set(SRC
 
 if (NGRAPH_ONNX_IMPORT_ENABLE)
     list(APPEND SRC onnx_import.cpp)
+    if (NGRAPH_ONNXIFI_ENABLE)
+        list(APPEND SRC onnxifi.cpp)
+    endif()
 endif()
 
 if (NGRAPH_INTERPRETER_ENABLE)
@@ -148,6 +151,11 @@ if (NGRAPH_GPU_ENABLE)
     target_link_libraries(unit-test gpu_backend)
 endif()
 
+if (NGRAPH_ONNXIFI_ENABLE)
+    target_include_directories(unit-test SYSTEM PUBLIC ${ONNX_INCLUDE_DIR})
+    target_link_libraries(unit-test onnxifi-ngraph)
+endif()
+
 add_custom_target(unit-test-check
     COMMAND ${PROJECT_BINARY_DIR}/test/unit-test \${ARGS}
     DEPENDS unit-test

test/onnxifi.cpp

+//*****************************************************************************
+// Copyright 2017-2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include <cstring>
+
+#include <gtest/gtest.h>
+#include <onnxifi.h>
+
+#include "ngraph/runtime/backend_manager.hpp"
+
+const constexpr std::size_t g_backend_ids_count{10};
+
+TEST(onnxifi, get_backend_ids)
+{
+    ::onnxBackendID backendIDs[g_backend_ids_count];
+    std::size_t count{g_backend_ids_count};
+    ::onnxStatus status{::onnxGetBackendIDs(backendIDs, &count)};
+    EXPECT_TRUE(status == ONNXIFI_STATUS_SUCCESS);
+    EXPECT_TRUE(count == ngraph::runtime::BackendManager::get_registered_backends().size());
+}
+
+TEST(onnxifi, get_backend_ids_buffer_null)
+{
+    std::size_t count{0};
+    ::onnxStatus status{::onnxGetBackendIDs(nullptr, &count)};
+    EXPECT_TRUE(status == ONNXIFI_STATUS_FALLBACK);
+    EXPECT_TRUE(count == ngraph::runtime::BackendManager::get_registered_backends().size());
+}
+
+TEST(onnxifi, get_backend_ids_count_null)
+{
+    ::onnxBackendID backendIDs[g_backend_ids_count];
+    ::onnxStatus status{::onnxGetBackendIDs(backendIDs, nullptr)};
+    EXPECT_TRUE(status == ONNXIFI_STATUS_INVALID_POINTER);
+}
+
+TEST(onnxifi, get_backend_ids_null)
+{
+    ::onnxStatus status{::onnxGetBackendIDs(nullptr, nullptr)};
+    EXPECT_TRUE(status == ONNXIFI_STATUS_INVALID_POINTER);
+}
+
+TEST(onnxifi, get_backend_ids_consistency_check)
+{
+    ::onnxBackendID first_ids[g_backend_ids_count];
+    std::size_t first_count{g_backend_ids_count};
+    EXPECT_TRUE(::onnxGetBackendIDs(first_ids, &first_count) == ONNXIFI_STATUS_SUCCESS);
+    EXPECT_TRUE(first_count == ngraph::runtime::BackendManager::get_registered_backends().size());
+    ::onnxBackendID second_ids[g_backend_ids_count];
+    std::size_t second_count{g_backend_ids_count};
+    EXPECT_TRUE(::onnxGetBackendIDs(second_ids, &second_count) == ONNXIFI_STATUS_SUCCESS);
+    EXPECT_TRUE(second_count == ngraph::runtime::BackendManager::get_registered_backends().size());
+    EXPECT_TRUE(first_count == second_count);
+    EXPECT_TRUE(std::memcmp(first_ids, second_ids, first_count) == 0);
+}