Add test revealing issue in current cudnn pooling implementation. (#2059)

* Add test revealing issue in current cudnn pooling operation. To fix we need op::MaxPoolBackprop to take the fprop output as an input argument.

* Formatting.

* Add comments and test description. Increase tensor size well past most cache sizes.

* exclude intelgpu

src/ngraph/runtime/gpu/unit_test.manifest

@@ -56,6 +56,8 @@ quantize_ROUND_TOWARD_INFINITY
 quantize_ROUND_TOWARD_ZERO
 quantize_ROUND_UP
 quantize_ROUND_DOWN
+# maxpool bug which requires maxpool backprop op change to fix
+maxpool_bprop_larger_than_cache
 shape_of_scalar
 shape_of_vector
 shape_of_matrix

src/ngraph/runtime/intelgpu/unit_test.manifest

@@ -31,6 +31,7 @@ dot_4d_5d_multi_axis_more
 generate_mask
 function_call
 max_pool_3d
+maxpool_bprop_larger_than_cache
 numeric_double_inf
 numeric_double_nan
 quantize

test/backend_pool.in.cpp

@@ -1235,3 +1235,84 @@ NGRAPH_TEST_P(${BACKEND_NAME}, avg_pool_3d_params, avg_pool_3d_uneven_strided_pa
 
 // avg_pool_3d case generation
 NGRAPH_INSTANTIATE_TEST_CASE_P(${BACKEND_NAME}, include_pad, avg_pool_3d_params, testing::Bool());
+
+//
+// This test primarly checks that maxpool backprop functions
+// correctly when the input tensor is larger than most cache sizes.
+// Here the to-be-pooled tensor is rank 2 with one non-trivial
+// dimension:
+//
+// x : [[0, 1, 0, 1, 0, 1, ... , 0, 1]]  <--- input data
+//       ----  ----  ----  ...   ----    <--- pooling windows
+// y : [[ 1  ,  1  ,  1  , ... ,  1]]    <--- max pooled output
+//
+// The pooling window is size 2 and stride 2, so the windows
+// do not overlap. Thus, each window will effectively see [0, 1]
+// as its input data for max pooling. The resulting output tensor
+// of pooling will be sizeof(x) with all elements equal to 1 as
+// seen above.
+// Therefore, for the backward pooling operation with the same window shape
+// and strides, the value of dy will only propogate to the positions in
+// dx that correspond to a value of 1 in the corresponding input tensor x:
+//
+// dy : [[2, 3, ... , 4]]
+// x  : [[0, 1, 0, 1, ... , 0, 1]]
+// dx : [[0, 2, 0, 3, ... , 0, 4]]
+//
+NGRAPH_TEST(${BACKEND_NAME}, maxpool_bprop_larger_than_cache)
+{
+    Shape window_shape{1, 2};
+    Strides move_strides{1, 2};
+    Shape padding_below{0, 0};
+    Shape padding_above{0, 0};
+
+    // 200 MB tensor to exceed cache
+    const size_t num_elements = 50 * 1024 * 1024;
+    auto ceil_div = [](size_t x, size_t y) { return 1 + ((x - 1) / y); };
+    const size_t num_pooled_elements = ceil_div(num_elements + padding_below.back() +
+                                                    padding_above.back() - window_shape.back() + 1,
+                                                move_strides.back());
+    Shape shape_x{1, 1, 1, num_elements};
+    Shape shape_y{1, 1, 1, num_pooled_elements};
+
+    auto x = make_shared<op::Parameter>(element::f32, shape_x);
+    auto dy = make_shared<op::Parameter>(element::f32, shape_y);
+    auto bprop =
+        make_shared<Function>(make_shared<op::MaxPoolBackprop>(
+                                  x, dy, window_shape, move_strides, padding_below, padding_above),
+                              op::ParameterVector{x, dy});
+
+    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+
+    // initialize x to array of alternating 0s and 1s as described above
+    std::vector<float> x_data(num_elements, 0);
+    for (auto i = 0u; i < num_elements; i++)
+    {
+        x_data[i] = (i % 2);
+    }
+    auto x_t = backend->create_tensor(element::f32, shape_x);
+    copy_data(x_t, x_data);
+
+    // use random data for deltas dy
+    std::vector<float> dy_data(num_pooled_elements);
+    test::Uniform<float> rng(0.0f, 1.0f);
+    rng.initialize(dy_data);
+    auto dy_t = backend->create_tensor(element::f32, shape_y);
+    copy_data(dy_t, dy_data);
+
+    // create result deltas tensor and run the backward max pooling operation
+    auto dx_t = backend->create_tensor(element::f32, shape_x);
+    backend->call_with_validate(bprop, {dx_t}, {x_t, dy_t});
+
+    // expected values should be dy with 0s left inserted
+    // for each delta, see test description above for details
+    std::vector<float> expected_dx(num_elements, 0);
+    for (auto i = 0u, j = 0u; i < num_elements; i++)
+    {
+        if (x_data[i])
+        {
+            expected_dx[i] = x_data[i] * dy_data[j++];
+        }
+    }
+    EXPECT_EQ(expected_dx, read_vector<float>(dx_t));
+}