gpu safe call - add CUDA_RT_SAFE_CALL (#1222)

* add CUDA_SAFE_CALL to all cuda calls

* add CUDA_RT_SAFE_CALL

* add null ptr check before free

* init pointer to nullptr

* consolidate conditions

src/ngraph/runtime/gpu/gpu_tensor_view.cpp

@@ -22,6 +22,7 @@
 #include "ngraph/descriptor/primary_tensor_view.hpp"
 #include "ngraph/runtime/gpu/gpu_backend.hpp"
 #include "ngraph/runtime/gpu/gpu_tensor_view.hpp"
+#include "ngraph/runtime/gpu/gpu_util.hpp"
 
 using namespace ngraph;
 using namespace std;
@@ -44,7 +45,7 @@ runtime::gpu::GPU_TensorView::GPU_TensorView(const ngraph::element::Type& elemen
     }
     else if (m_buffer_size > 0)
     {
-        cudaMalloc(static_cast<void**>(&m_allocated_buffer_pool), m_buffer_size);
+        CUDA_RT_SAFE_CALL(cudaMalloc(static_cast<void**>(&m_allocated_buffer_pool), m_buffer_size));
     }
 }
 
@@ -56,18 +57,18 @@ runtime::gpu::GPU_TensorView::GPU_TensorView(const ngraph::element::Type& elemen
 
 runtime::gpu::GPU_TensorView::~GPU_TensorView()
 {
-    if (!m_custom_memory)
+    if (!m_custom_memory && (m_allocated_buffer_pool != nullptr))
     {
-        cudaFree(m_allocated_buffer_pool);
+        CUDA_RT_SAFE_CALL(cudaFree(m_allocated_buffer_pool));
     }
 }
 
 void runtime::gpu::GPU_TensorView::write(const void* source, size_t tensor_offset, size_t n)
 {
-    cudaMemcpy(m_allocated_buffer_pool, source, n, cudaMemcpyHostToDevice);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(m_allocated_buffer_pool, source, n, cudaMemcpyHostToDevice));
 }
 
 void runtime::gpu::GPU_TensorView::read(void* target, size_t tensor_offset, size_t n) const
 {
-    cudaMemcpy(target, m_allocated_buffer_pool, n, cudaMemcpyDeviceToHost);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(target, m_allocated_buffer_pool, n, cudaMemcpyDeviceToHost));
 }

src/ngraph/runtime/gpu/gpu_tensor_view.hpp

@@ -54,7 +54,7 @@ public:
     /// @param n Number of bytes to read, must be integral number of elements.
     void read(void* p, size_t tensor_offset, size_t n) const override;
 
-    void* m_allocated_buffer_pool;
+    void* m_allocated_buffer_pool = nullptr;
     size_t m_buffer_size;
     bool m_custom_memory;
 };

src/ngraph/runtime/gpu/gpu_util.cpp

@@ -34,7 +34,7 @@ void runtime::gpu::print_gpu_f32_tensor(const void* p, size_t element_count, siz
 {
     std::vector<float> local(element_count);
     size_t size_in_bytes = element_size * element_count;
-    cudaMemcpy(local.data(), p, size_in_bytes, cudaMemcpyDeviceToHost);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(local.data(), p, size_in_bytes, cudaMemcpyDeviceToHost));
     std::cout << "{" << join(local) << "}" << std::endl;
 }
 
@@ -46,7 +46,7 @@ void runtime::gpu::check_cuda_errors(CUresult err)
 void* runtime::gpu::create_gpu_buffer(size_t buffer_size, const void* data)
 {
     void* allocated_buffer_pool;
-    cudaMalloc(static_cast<void**>(&allocated_buffer_pool), buffer_size);
+    CUDA_RT_SAFE_CALL(cudaMalloc(static_cast<void**>(&allocated_buffer_pool), buffer_size));
     if (data)
     {
         runtime::gpu::cuda_memcpyHtD(allocated_buffer_pool, data, buffer_size);
@@ -58,28 +58,28 @@ void runtime::gpu::free_gpu_buffer(void* buffer)
 {
     if (buffer)
     {
-        cudaFree(buffer);
+        CUDA_RT_SAFE_CALL(cudaFree(buffer));
     }
 }
 
 void runtime::gpu::cuda_memcpyDtD(void* dst, const void* src, size_t buffer_size)
 {
-    cudaMemcpy(dst, src, buffer_size, cudaMemcpyDeviceToDevice);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(dst, src, buffer_size, cudaMemcpyDeviceToDevice));
 }
 
 void runtime::gpu::cuda_memcpyHtD(void* dst, const void* src, size_t buffer_size)
 {
-    cudaMemcpy(dst, src, buffer_size, cudaMemcpyHostToDevice);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(dst, src, buffer_size, cudaMemcpyHostToDevice));
 }
 
 void runtime::gpu::cuda_memcpyDtH(void* dst, const void* src, size_t buffer_size)
 {
-    cudaMemcpy(dst, src, buffer_size, cudaMemcpyDeviceToHost);
+    CUDA_RT_SAFE_CALL(cudaMemcpy(dst, src, buffer_size, cudaMemcpyDeviceToHost));
 }
 
 void runtime::gpu::cuda_memset(void* dst, int value, size_t buffer_size)
 {
-    cudaMemset(dst, value, buffer_size);
+    CUDA_RT_SAFE_CALL(cudaMemset(dst, value, buffer_size));
 }
 
 namespace

src/ngraph/runtime/gpu/gpu_util.hpp

@@ -60,6 +60,20 @@
         }                                                                                          \
     } while (0)
 
+#define CUDA_RT_SAFE_CALL(x)                                                                       \
+    do                                                                                             \
+    {                                                                                              \
+        cudaError_t err = x;                                                                       \
+        if (cudaSuccess != err)                                                                    \
+        {                                                                                          \
+            std::stringstream safe_call_ss;                                                        \
+            safe_call_ss << "\nerror: " #x " failed with error"                                    \
+                         << "\nfile: " << __FILE__ << "\nline: " << __LINE__                       \
+                         << "\nmsg: " << cudaGetErrorString(err);                                  \
+            throw std::runtime_error(safe_call_ss.str());                                          \
+        }                                                                                          \
+    } while (0)
+
 #define CUDNN_SAFE_CALL(func)                                                                      \
     do                                                                                             \
     {                                                                                              \