autodiff.in.cpp 69.9 KB
Newer Older
1
//*****************************************************************************
2
// Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
16 17

#include <algorithm>
18
#include <functional>
19
#include <memory>
20
#include <tuple>
21 22 23

#include "gtest/gtest.h"

24 25 26 27
// clang-format off
#define AUTODIFF_BACKEND_${BACKEND_NAME}
// clang-format on

28
#include "ngraph/ngraph.hpp"
29
#include "ngraph/pass/manager.hpp"
30
#if defined(AUTODIFF_BACKEND_CPU)
31
#include "ngraph/runtime/cpu/op/batch_mat_mul_transpose.hpp"
32
#include "ngraph/runtime/cpu/pass/cpu_mat_fusion.hpp"
33
#endif
34
#include "ngraph/runtime/reference/avg_pool.hpp"
35
#include "util/autodiff/backprop_function.hpp"
adstraw's avatar
adstraw committed
36
#include "util/autodiff/numeric_compare.hpp"
37
#include "util/random.hpp"
38
#include "util/test_control.hpp"
39 40 41
using namespace std;
using namespace ngraph;

42 43 44
static string s_manifest = "${MANIFEST}";

NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4_c1_hw4_2x2_max)
45
{
46
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
47

48
    Shape shape_a{1, 4, 4, 4}; // in CHWN
49
    Shape maxpool_shape{1, 4, 3, 3};
50 51 52

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
53
        A, AxisVector{0, 3, 1, 2}, Shape{1, 4, 4, 4}); // convert CHWN to CNHW
54
    Shape window_shape{2, 2};
55 56
    auto window_movement_strides = Strides{1, 1};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
57
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
58

59
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
60 61
    vector<int> dataEp(shape_size(maxpool_shape), 4);

62 63
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
64 65 66 67 68 69

    vector<int> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
                          13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                          17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                          45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};

70
    vector<int> expected{// delta
71 72 73 74 75 76 77 78 79
                         0, 4, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 4,  4, 12, 0,
                         0, 0, 0, 8, 0, 0, 4, 8, 0, 8, 0, 0, 8, 0, 0, 0, 0, 4, 16, 4, 16, 8,
                         0, 0, 0, 4, 0, 4, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
80
    auto handle = backend->compile(df);
81
    handle->call_with_validate({output}, {input, ep});
82
    ASSERT_TRUE(read_vector<int>(output) == expected);
83 84
}

85
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max)
86
{
87
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
88

89
    Shape shape_a{1, 5, 5, 2}; // in CHWN
90
    Shape maxpool_shape{1, 2, 2, 2};
91 92 93

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
94
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
95
    Shape window_shape{3, 3};
96 97
    auto window_movement_strides = Strides{2, 2};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
98
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
99

100
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
101 102
    vector<int> dataEp(shape_size(maxpool_shape), 4);

103 104
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
105 106 107 108 109

    vector<int> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                          30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                          19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

110
    vector<int> expected{// delta
111 112 113 114 115 116 117 118
                         4, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 4, 4, 0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
119
    auto handle = backend->compile(df);
120
    handle->call_with_validate({output}, {input, ep});
121
    ASSERT_TRUE(read_vector<int>(output) == expected);
122 123
}

124 125 126 127
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max_pad1x2_2x3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

128
    Shape shape_a{1, 5, 5, 2}; // in CHWN
129 130 131 132
    Shape maxpool_shape{1, 2, 4, 5};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    auto reshape = make_shared<op::Reshape>(
133
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
134 135 136 137 138 139
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
    Shape pad_below{1, 2};
    Shape pad_above{3, 4};
    auto maxpool = make_shared<op::MaxPool>(
        reshape, window_shape, window_movement_strides, pad_below, pad_above);
140
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
141

142
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
143 144
    vector<float> dataEp(shape_size(maxpool_shape), 4);

145 146
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
147 148 149 150 151

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

152
    vector<float> expected{// delta
153 154 155 156 157 158 159 160
                           8, 0, 0, 0, 0, 4,  0, 0, 8, 0, 0, 8, 4, 8, 0, 0, 0,
                           0, 0, 4, 4, 0, 0,  0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                           0, 0, 0, 0, 4, 12, 4, 8, 4, 0, 0, 0, 0, 4, 8, 0};
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
161
    auto handle = backend->compile(df);
162
    handle->call_with_validate({output}, {input, ep});
163 164 165
    EXPECT_EQ(expected, read_vector<float>(output));
}

166
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw2x2)
167
{
168
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
169

170
    Shape padding{1, 1};
171

172 173
    Shape shape_a{1, 1, 2, 2};
    Shape avgpool_shape{1, 1, 2, 2};
174 175

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
176
    Shape window_shape{2, 2};
177 178
    auto window_movement_strides = Strides{2, 2};
    auto avgpool =
179
        make_shared<op::AvgPool>(A, window_shape, window_movement_strides, padding, padding, false);
180
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
181

182
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
183 184
    vector<int> dataEp(shape_size(avgpool_shape), 4);

185
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
186

187
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
188 189 190 191 192 193 194 195 196 197

    vector<int> dataInput{4, 8, 12, 16};

    vector<int> expected{1, 2, 3, 4};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
198
    auto handle = backend->compile(df);
199
    handle->call_with_validate({output}, {input, ep});
200 201 202
    ASSERT_TRUE(read_vector<int>(output) == dataEp);
}

203
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw4x4)
204
{
205
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
206

207 208
    Shape shape_a{1, 1, 4, 4};
    Shape avgpool_shape{1, 1, 3, 3};
209 210

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
211
    Shape window_shape{2, 2};
212 213
    auto window_movement_strides = Strides{1, 1};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
214
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
215

216
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
217 218
    vector<int> dataEp(shape_size(avgpool_shape), 4);

219
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
220

221
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
222 223 224 225 226 227 228 229 230 231

    vector<int> dataInput{1, 3, 1, 3, 1, 3, 1, 3, 3, 5, 3, 5, 3, 5, 3, 5};

    vector<int> expected{1, 2, 2, 1, 2, 4, 4, 2, 2, 4, 4, 2, 1, 2, 2, 1};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
232
    auto handle = backend->compile(df);
233
    handle->call_with_validate({output}, {input, ep});
234 235 236
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

237
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4)
238
{
239
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
240

241 242
    Shape shape_a{2, 2, 4, 4};
    Shape avgpool_shape{2, 2, 2, 2};
243 244

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
245
    Shape window_shape{2, 2};
246 247
    auto window_movement_strides = Strides{2, 2};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
248
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
249

250
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
251 252
    vector<int> dataEp(shape_size(avgpool_shape), 12);

253
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
254

255
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
256

257
    vector<int> dataInput{// i1c1
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
                          1,
                          2,
                          6,
                          7,
                          3,
                          4,
                          4,
                          3,
                          19,
                          1,
                          2,
                          3,
                          18,
                          2,
                          3,
                          2,
274
                          // i1c2
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
                          4,
                          1,
                          5,
                          5,
                          1,
                          4,
                          5,
                          5,
                          12,
                          8,
                          2,
                          3,
                          15,
                          5,
                          3,
                          2,
291
                          // i2c1
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
                          2,
                          3,
                          7,
                          7,
                          3,
                          2,
                          3,
                          3,
                          13,
                          7,
                          1,
                          2,
                          7,
                          13,
                          3,
                          4,
308
                          // i2c2
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
                          1,
                          1,
                          2,
                          2,
                          7,
                          1,
                          2,
                          14,
                          6,
                          16,
                          4,
                          1,
                          14,
                          4,
                          4,
                          1};

    vector<int> expected(shape_size(shape_a), 3);
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
332
    auto handle = backend->compile(df);
333
    handle->call_with_validate({output}, {input, ep});
334 335 336
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

337
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_numeric)
338
{
339
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
340
    Shape shape_a{2, 2, 4, 4};
341 342 343 344
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
345
        Shape window_shape{2, 2};
346 347
        auto window_movement_strides = Strides{2, 2};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
348
        return make_shared<Function>(avgpool, ParameterVector{A});
349 350 351

    };

352 353
    auto f = make_graph();
    auto g = make_graph();
354
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
355
    {
356
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
357
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
358 359 360
    }
}

361
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_win_2x2_str_1x1_numeric)
362
{
363
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
364
    Shape shape_a{2, 2, 4, 4};
365 366 367 368
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
369
        Shape window_shape{2, 2};
370 371
        auto window_movement_strides = Strides{1, 1};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
372
        return make_shared<Function>(avgpool, ParameterVector{A});
373 374 375

    };

376 377
    auto f = make_graph();
    auto g = make_graph();
378
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
379
    {
380
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
381
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
382 383 384
    }
}

385
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw2x2_win_2x2_str_1x1_padding_numeric)
386
{
387
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
388
    Shape shape_a{2, 2, 4, 4};
389 390 391 392
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
393 394
        Shape window_shape{2, 2};
        Shape padding{1, 1};
395
        auto window_movement_strides = Strides{2, 2};
396 397
        auto avgpool = make_shared<op::AvgPool>(
            A, window_shape, window_movement_strides, padding, padding, false);
398
        return make_shared<Function>(avgpool, ParameterVector{A});
399 400 401

    };

402 403
    auto f = make_graph();
    auto g = make_graph();
404
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
405
    {
406
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
407
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
408 409 410
    }
}

411
NGRAPH_TEST(${BACKEND_NAME}, backwards_abs)
412
{
413
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
414 415 416 417 418

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
419
    Shape shape{2, 3};
420 421

    auto make_graph = [shape]() {
422
        auto X = make_shared<op::Parameter>(element::f32, shape);
423 424
        return make_shared<Function>(make_shared<op::Abs>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
425 426
    };

427 428
    auto f = make_graph();
    auto g = make_graph();
429
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
430
    {
431
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
432

433
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
434

435
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
436

437
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
438
    }
439 440
}

441
NGRAPH_TEST(${BACKEND_NAME}, backwards_acos)
442 443 444 445 446 447 448 449 450 451 452 453
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Acos>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
454
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
455 456
}

457
NGRAPH_TEST(${BACKEND_NAME}, backwards_add)
458
{
459
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
460

461
    test::Uniform<float> rng(-1.0f, 1.0f);
462
    Shape shape{2, 3};
463 464
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
465

466
    auto make_graph = [shape]() {
467 468
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
469
        return make_shared<Function>(X0 + X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
470
    };
471
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
472 473
}

474
NGRAPH_TEST(${BACKEND_NAME}, backwards_add_nested)
475
{
476
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
477

478
    test::Uniform<float> rng(-1.0f, 1.0f);
479
    Shape shape{2, 3};
480 481
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
482 483

    auto make_graph = [shape]() {
484 485
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
486 487
        return make_shared<Function>((X0 + X1) + (X1 + X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
488
    };
489
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
490 491
}

492
NGRAPH_TEST(${BACKEND_NAME}, backwards_asin)
493 494 495 496 497 498 499 500 501 502 503 504
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Asin>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
505
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
506 507
}

508
NGRAPH_TEST(${BACKEND_NAME}, backwards_atan)
509 510 511 512 513 514 515 516 517 518 519 520
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-10.0f, 10.0f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Atan>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
521
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
522 523
}

524
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast0)
525
{
526
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
527

528
    test::Uniform<float> rng(-1.0f, 1.0f);
529
    Shape shape{3};
530
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
531 532

    auto make_graph = [shape]() {
533
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
534 535 536
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{2, 3}, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
537
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
538 539
}

540
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast1)
541
{
542
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
543

544
    test::Uniform<float> rng(-1.0f, 1.0f);
545
    Shape shape{3};
546
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
547 548

    auto make_graph = [shape]() {
549
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
550 551 552
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{3, 2}, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
553
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
554 555
}

556
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_vector)
557
{
558
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
559 560

    test::Uniform<float> rng(-1.0f, 1.0f);
561
    Shape shape_0{3};
562
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
563
    Shape shape_1{2};
564
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
565
    Shape shape_2{1};
566
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
567 568

    auto make_graph = [shape_0, shape_1, shape_2]() {
569 570 571
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
572
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
573 574
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
575 576
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
577 578
}

579
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_0)
580
{
581
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
582 583

    test::Uniform<float> rng(-1.0f, 1.0f);
584
    Shape shape_0{3, 2};
585
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
586
    Shape shape_1{2, 2};
587
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
588
    Shape shape_2{1, 2};
589
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
590 591

    auto make_graph = [shape_0, shape_1, shape_2]() {
592 593 594
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
595
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
596 597
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
598 599
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
600 601
}

602
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_1)
603
{
604
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
605 606

    test::Uniform<float> rng(-1.0f, 1.0f);
607
    Shape shape_0{2, 3};
608
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
609
    Shape shape_1{2, 2};
610
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
611
    Shape shape_2{2, 1};
612
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
613 614

    auto make_graph = [shape_0, shape_1, shape_2]() {
615 616 617
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
618
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 1),
619 620
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
621 622
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
623 624
}

625
NGRAPH_TEST(${BACKEND_NAME}, backwards_ceiling)
626
{
627
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
628 629 630 631 632 633

    // The numeric derivative and the symbolic one may disagree near integers, so we will dance around
    // them.
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
634
    Shape shape{2, 3};
635 636

    auto make_graph = [shape]() {
637
        auto X = make_shared<op::Parameter>(element::f32, shape);
638 639
        return make_shared<Function>(make_shared<op::Ceiling>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
640 641
    };

642 643
    auto f = make_graph();
    auto g = make_graph();
644
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
645
    {
646
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
647

648
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
649

650
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
651

652
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
653

654
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
655

656
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
657 658 659
    }
}

660
NGRAPH_TEST(${BACKEND_NAME}, backwards_cos)
661
{
662
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
663 664

    test::Uniform<float> rng(-10.0f, 10.0f);
665
    Shape shape{2, 3};
666
    auto make_graph = [shape]() {
667
        auto X = make_shared<op::Parameter>(element::f32, shape);
668 669
        return make_shared<Function>(make_shared<op::Cos>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
670 671
    };

672 673
    auto f = make_graph();
    auto g = make_graph();
674
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
675
    {
676
        auto x = rng.initialize(backend->create_tensor<float>(shape));
677

678
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
679 680 681
    }
}

682
NGRAPH_TEST(${BACKEND_NAME}, backwards_cosh)
683
{
684
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
685 686

    test::Uniform<float> rng(-10.0f, 10.0f);
687
    Shape shape{2, 3};
688
    auto make_graph = [shape]() {
689
        auto X = make_shared<op::Parameter>(element::f32, shape);
690 691
        return make_shared<Function>(make_shared<op::Cosh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
692 693
    };

694 695
    auto f = make_graph();
    auto g = make_graph();
696
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
697
    {
698
        auto x = rng.initialize(backend->create_tensor<float>(shape));
699

700
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
701 702 703
    }
}

704
NGRAPH_TEST(${BACKEND_NAME}, backwards_divide)
705
{
706
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
707

708 709 710
    test::Uniform<float> rng(-1.0f, 1.0f);
    test::Uniform<float> rng1(1.0f, 2.0f);
    test::Uniform<float> rng2(-2.0f, -1.0f);
711
    Shape shape{2, 3};
712 713 714
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng1.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng2.initialize(backend->create_tensor<float>(shape));
715

716
    auto make_graph = [shape]() {
717 718
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
719
        return make_shared<Function>(X0 / X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
720
    };
721 722
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
723
}
724

725
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_scalar)
726
{
727
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
728

729
    test::Uniform<float> rng(-1.0f, 1.0f);
730 731
    Shape shape0{};
    Shape shape1{};
732 733
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
734 735

    auto make_graph = [shape0, shape1]() {
736 737
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
738 739 740
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
741
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
742 743
}

744
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_tensor)
745
{
746
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
747

748
    test::Uniform<float> rng(-1.0f, 1.0f);
749 750
    Shape shape0{};
    Shape shape1{3, 4};
751 752
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
753 754

    auto make_graph = [shape0, shape1]() {
755 756
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
757 758 759
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
760
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
761 762
}

763
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_scalar)
764
{
765
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
766

767
    test::Uniform<float> rng(-1.0f, 1.0f);
768 769
    Shape shape0{3, 4};
    Shape shape1{};
770 771
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
772 773

    auto make_graph = [shape0, shape1]() {
774 775
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
776 777 778
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
779
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
780 781
}

782
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_vector_vector)
783
{
784
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
785

786
    test::Uniform<float> rng(-1.0f, 1.0f);
787 788
    Shape shape0{3};
    Shape shape1{3};
789 790
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
791 792

    auto make_graph = [shape0, shape1]() {
793 794
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
795 796 797
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
798
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
799 800
}

801
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_vector)
802
{
803
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
804

805
    test::Uniform<float> rng(-1.0f, 1.0f);
806 807
    Shape shape0{4, 3};
    Shape shape1{3};
808 809
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
810 811

    auto make_graph = [shape0, shape1]() {
812 813
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
814 815 816
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
817
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
818 819
}

820
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor2_tensor2)
821
{
822
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
823

824
    test::Uniform<float> rng(-1.0f, 1.0f);
825 826
    Shape shape0{4, 3};
    Shape shape1{3, 5};
827 828
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
829 830

    auto make_graph = [shape0, shape1]() {
831 832
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
833 834
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Adam Procter's avatar
Adam Procter committed
835
    };
836
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Adam Procter's avatar
Adam Procter committed
837 838
}

839
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor3_tensor3)
Adam Procter's avatar
Adam Procter committed
840
{
841
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Adam Procter's avatar
Adam Procter committed
842 843

    test::Uniform<float> rng(-1.0f, 1.0f);
844 845
    Shape shape0{2, 4, 3};
    Shape shape1{4, 3, 3};
846 847
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
Adam Procter's avatar
Adam Procter committed
848 849

    auto make_graph = [shape0, shape1]() {
850 851
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
Adam Procter's avatar
Adam Procter committed
852 853
        return make_shared<Function>(make_shared<op::Dot>(X0, X1, 2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
854
    };
855
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
856 857
}

858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
NGRAPH_TEST(${BACKEND_NAME}, backwards_batchmatmul_tensor2_tensor2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape0{3, 4, 5};
    Shape shape1{3, 5, 6};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));

    auto make_graph = [shape0, shape1]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
        return make_shared<Function>(make_shared<op::BatchMatMul>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };

    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
}
876

877
NGRAPH_TEST(${BACKEND_NAME}, backwards_exp)
Scott Cyphers's avatar
Scott Cyphers committed
878
{
879
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
880

881
    test::Uniform<float> rng(-1.0f, 1.0f);
882
    Shape shape{2, 3};
883
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
884 885

    auto make_graph = [shape]() {
886
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
887 888
        return make_shared<Function>(make_shared<op::Exp>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
889
    };
890
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
891 892
}

893
NGRAPH_TEST(${BACKEND_NAME}, backwards_floor)
894
{
895
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
896 897 898 899 900 901

    // The numeric derivative and the symbolic one may disagree near integers, so we will dance around
    // them.
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
902
    Shape shape{2, 3};
903 904

    auto make_graph = [shape]() {
905
        auto X = make_shared<op::Parameter>(element::f32, shape);
906 907
        return make_shared<Function>(make_shared<op::Floor>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
908 909
    };

910 911
    auto f = make_graph();
    auto g = make_graph();
912
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
913
    {
914
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
915

916
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
917

918
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
919

920
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
921

922
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
923

924
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
925 926 927
    }
}

928
NGRAPH_TEST(${BACKEND_NAME}, backwards_log)
Scott Cyphers's avatar
Scott Cyphers committed
929
{
930
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
931

932
    test::Uniform<float> rng(1.0f, 2.0f);
933
    Shape shape{2, 3};
934
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
935 936

    auto make_graph = [shape]() {
937
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
938 939
        return make_shared<Function>(make_shared<op::Log>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
940
    };
941
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
942 943
}

944
NGRAPH_TEST(${BACKEND_NAME}, backwards_maximum)
945
{
946
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
947

948
    test::Uniform<float> rng(-1.0f, 1.0f);
949
    Shape shape{2, 3};
950 951
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
952 953

    auto make_graph = [shape]() {
954 955
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
956 957 958
        return make_shared<Function>(make_shared<op::Maximum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
959
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
960 961
}

962
NGRAPH_TEST(${BACKEND_NAME}, backwards_minimum)
963
{
964
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
965

966
    test::Uniform<float> rng(-1.0f, 1.0f);
967
    Shape shape{2, 3};
968 969
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
970 971

    auto make_graph = [shape]() {
972 973
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
974 975 976
        return make_shared<Function>(make_shared<op::Minimum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
977
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
978 979
}

980
NGRAPH_TEST(${BACKEND_NAME}, backwards_multiply)
981
{
982
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
983

984
    test::Uniform<float> rng(-1.0f, 1.0f);
985
    Shape shape{2, 3};
986 987
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
988

989
    auto make_graph = [shape]() {
990 991
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
992
        return make_shared<Function>(X0 * X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
993
    };
994
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
995
}
Scott Cyphers's avatar
Scott Cyphers committed
996

997
NGRAPH_TEST(${BACKEND_NAME}, backwards_negative)
Scott Cyphers's avatar
Scott Cyphers committed
998
{
999
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1000

1001
    test::Uniform<float> rng(-1.0f, 1.0f);
1002
    Shape shape{2, 3};
1003
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1004 1005

    auto make_graph = [shape]() {
1006
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1007
        return make_shared<Function>(-X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1008
    };
1009
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1010 1011
}

1012
NGRAPH_TEST(${BACKEND_NAME}, backwards_parameter)
Scott Cyphers's avatar
Scott Cyphers committed
1013
{
1014
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1015

1016
    test::Uniform<float> rng(-1.0f, 1.0f);
1017
    Shape shape{2, 3};
1018
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1019
    auto make_graph = [shape]() {
1020
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1021
        return make_shared<Function>(X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1022
    };
1023
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
1024 1025
}

1026
NGRAPH_TEST(${BACKEND_NAME}, backwards_power)
1027
{
1028
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1029 1030 1031

    test::Uniform<float> rng_neg(-5.0f, -0.5f);
    test::Uniform<float> rng_pos(0.5f, 5.0f);
1032
    Shape shape{2, 3};
1033 1034

    auto make_graph = [shape]() {
1035 1036
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1037 1038 1039 1040
        return make_shared<Function>(std::make_shared<op::Power>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };

1041 1042
    auto x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_neg.initialize(backend->create_tensor<float>(shape));
1043

1044
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1045

1046 1047
    x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1048

1049
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1050 1051
}

1052
NGRAPH_TEST(${BACKEND_NAME}, backwards_relu)
1053
{
1054
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1055 1056 1057 1058

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
1059 1060
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1061 1062 1063 1064 1065 1066 1067

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Relu>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1068 1069
    auto f = make_graph();
    auto g = make_graph();
1070 1071
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
1072
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1073

1074
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1075

1076
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1077

1078
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1079 1080 1081
    }
}

1082
NGRAPH_TEST(${BACKEND_NAME}, backwards_replace_slice)
1083
{
1084
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1085 1086

    test::Uniform<float> rng(-10.0f, 10.0f);
1087 1088
    Shape shape_x{5, 5};
    Shape shape_y{2, 2};
1089
    auto make_graph = [shape_x, shape_y]() {
1090 1091
        auto X = make_shared<op::Parameter>(element::f32, shape_x);
        auto Y = make_shared<op::Parameter>(element::f32, shape_y);
1092 1093 1094 1095 1096
        return make_shared<Function>(
            make_shared<op::ReplaceSlice>(X, Y, Coordinate{2, 3}, Coordinate{4, 5}),
            std::vector<std::shared_ptr<op::Parameter>>{X, Y});
    };

1097 1098
    auto f = make_graph();
    auto g = make_graph();
1099
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1100
    {
1101 1102
        auto x = rng.initialize(backend->create_tensor<float>(shape_x));
        auto y = rng.initialize(backend->create_tensor<float>(shape_y));
1103

1104
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x, y}, .01f, .01f));
1105 1106 1107
    }
}

1108
NGRAPH_TEST(${BACKEND_NAME}, backwards_reshape)
1109
{
1110
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1111

1112
    test::Uniform<float> rng(-1.0f, 1.0f);
1113
    Shape shape{3, 4};
1114
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
1115 1116

    auto make_graph = [shape]() {
1117
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1118 1119 1120
        return make_shared<Function>(make_shared<op::Reshape>(X0, AxisVector{1, 0}, Shape{4, 3}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1121
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1122 1123
}

1124
NGRAPH_TEST(${BACKEND_NAME}, backwards_select)
1125
{
1126
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1127 1128

    test::Uniform<float> rng(-10.0f, 10.0f);
1129
    Shape shape{2, 3};
1130
    auto make_graph = [shape]() {
1131 1132 1133
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1134 1135 1136 1137
        return make_shared<Function>(make_shared<op::Select>(X0, X1, X2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1138 1139
    auto f = make_graph();
    auto g = make_graph();
1140
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1141
    {
1142
        auto x0 = backend->create_tensor(element::boolean, shape);
1143
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1144 1145 1146 1147
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1148
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1149 1150 1151
    }
}

1152
NGRAPH_TEST(${BACKEND_NAME}, backwards_select_nested)
1153
{
1154
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1155 1156

    test::Uniform<float> rng(-10.0f, 10.0f);
1157
    Shape shape{2, 3};
1158
    auto make_graph = [shape]() {
1159 1160 1161
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1162
        return make_shared<Function>(make_shared<op::Select>(X0, X2 + X1, X2 - X1),
1163 1164 1165
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1166 1167
    auto f = make_graph();
    auto g = make_graph();
1168
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1169
    {
1170
        auto x0 = backend->create_tensor(element::boolean, shape);
1171
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1172 1173 1174 1175
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1176
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1177 1178 1179
    }
}

1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
NGRAPH_TEST(${BACKEND_NAME}, backwards_sigmoid)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Sigmoid>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1196 1197
    auto f = make_graph();
    auto g = make_graph();
1198 1199 1200 1201
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));

1202
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1203 1204 1205

        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));

1206
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1207 1208 1209
    }
}

1210
NGRAPH_TEST(${BACKEND_NAME}, backwards_sign)
1211
{
1212
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1213 1214 1215 1216 1217

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
1218
    Shape shape{2, 3};
1219 1220

    auto make_graph = [shape]() {
1221
        auto X = make_shared<op::Parameter>(element::f32, shape);
1222 1223
        return make_shared<Function>(make_shared<op::Sign>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1224 1225
    };

1226 1227
    auto f = make_graph();
    auto g = make_graph();
1228
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1229
    {
1230
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1231

1232
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1233

1234
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1235

1236
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1237 1238 1239
    }
}

1240
NGRAPH_TEST(${BACKEND_NAME}, backwards_sin)
1241
{
1242
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1243 1244

    test::Uniform<float> rng(-10.0f, 10.0f);
1245
    Shape shape{2, 3};
1246
    auto make_graph = [shape]() {
1247
        auto X = make_shared<op::Parameter>(element::f32, shape);
1248 1249
        return make_shared<Function>(make_shared<op::Sin>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1250 1251
    };

1252 1253
    auto f = make_graph();
    auto g = make_graph();
1254
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1255
    {
1256
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1257

1258
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1259 1260 1261
    }
}

1262
NGRAPH_TEST(${BACKEND_NAME}, backwards_sinh)
1263
{
1264
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1265 1266

    test::Uniform<float> rng(-10.0f, 10.0f);
1267
    Shape shape{2, 3};
1268
    auto make_graph = [shape]() {
1269
        auto X = make_shared<op::Parameter>(element::f32, shape);
1270 1271
        return make_shared<Function>(make_shared<op::Sinh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1272 1273
    };

1274 1275
    auto f = make_graph();
    auto g = make_graph();
1276
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1277
    {
1278
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1279

1280
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1281 1282 1283
    }
}

1284
NGRAPH_TEST(${BACKEND_NAME}, backwards_slice)
1285
{
1286
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1287
    test::Uniform<float> rng(-10.0f, 10.0f);
1288
    Shape shape{5, 5};
1289
    auto make_graph = [shape]() {
1290
        auto X = make_shared<op::Parameter>(element::f32, shape);
1291 1292 1293 1294
        return make_shared<Function>(make_shared<op::Slice>(X, Coordinate{2, 3}, Coordinate{4, 5}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1295 1296
    auto f = make_graph();
    auto g = make_graph();
1297
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1298
    {
1299
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1300

1301
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1302 1303 1304
    }
}

1305
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_all)
1306
{
1307
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1308

adstraw's avatar
adstraw committed
1309
    test::Uniform<float> rng(-1.0f, 1.0f);
1310
    Shape shape{2, 3};
1311
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1312

1313
    auto make_graph = [shape]() {
adstraw's avatar
adstraw committed
1314 1315 1316
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
1317
    };
1318
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1319
}
1320

1321
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_axis)
adstraw's avatar
adstraw committed
1322
{
1323
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1324

adstraw's avatar
adstraw committed
1325 1326
    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3};
1327
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1328 1329 1330 1331 1332 1333

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1334
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1335 1336
}

1337
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_underflow)
adstraw's avatar
adstraw committed
1338
{
1339
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1340 1341 1342 1343

    auto low = std::numeric_limits<float>::lowest();

    Shape shape{2, 3};
1344
    auto x0 = backend->create_tensor(element::f32, shape);
adstraw's avatar
adstraw committed
1345 1346 1347 1348 1349 1350 1351
    copy_data(x0, vector<float>{low, 1, 2, 3, 4, 5});

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1352
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1353 1354
}

1355
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_3d)
adstraw's avatar
adstraw committed
1356
{
1357
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1358 1359 1360

    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3, 4};
1361
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1362 1363 1364 1365 1366 1367

    auto make_graph0 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1368
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph0, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1369 1370 1371 1372 1373 1374

    auto make_graph1 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1375
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph1, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1376 1377 1378 1379 1380 1381

    auto make_graph2 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1382
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph2, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1383 1384 1385 1386 1387 1388

    auto make_graph01 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1389
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph01, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1390 1391 1392 1393 1394 1395

    auto make_graph02 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1396
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph02, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1397 1398 1399 1400 1401 1402

    auto make_graph12 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1403
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph12, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1404 1405 1406 1407 1408 1409

    auto make_graph012 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1410
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph012, {x0}, .01f, .01f));
1411 1412
}

1413
NGRAPH_TEST(${BACKEND_NAME}, backwards_subtract)
Scott Cyphers's avatar
Scott Cyphers committed
1414
{
1415
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1416

1417
    test::Uniform<float> rng(-1.0f, 1.0f);
1418
    Shape shape{2, 3};
1419 1420
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1421 1422

    auto make_graph = [shape]() {
1423 1424
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1425
        return make_shared<Function>(X0 - X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Scott Cyphers's avatar
Scott Cyphers committed
1426
    };
1427
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1428
}
Scott Cyphers's avatar
Scott Cyphers committed
1429

1430
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_v2s)
1431
{
1432
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1433 1434

    test::Uniform<float> rng(-1.0f, 1.0f);
1435
    Shape shape{8};
1436
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1437 1438

    auto make_graph = [shape]() {
1439
        auto X = make_shared<op::Parameter>(element::f32, shape);
1440 1441 1442
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1443
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1444 1445
}

1446
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2s)
1447
{
1448
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1449 1450

    test::Uniform<float> rng(-1.0f, 1.0f);
1451
    Shape shape{8, 9};
1452
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1453 1454

    auto make_graph = [shape]() {
1455
        auto X = make_shared<op::Parameter>(element::f32, shape);
1456 1457 1458
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1459
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1460 1461
}

1462
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_0)
1463
{
1464
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1465 1466

    test::Uniform<float> rng(-1.0f, 1.0f);
1467
    Shape shape{8, 9};
1468
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1469 1470

    auto make_graph = [shape]() {
1471
        auto X = make_shared<op::Parameter>(element::f32, shape);
1472 1473 1474
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1475
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1476 1477
}

1478
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_1)
1479
{
1480
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1481 1482

    test::Uniform<float> rng(-1.0f, 1.0f);
1483
    Shape shape{8, 9};
1484
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1485 1486

    auto make_graph = [shape]() {
1487
        auto X = make_shared<op::Parameter>(element::f32, shape);
1488 1489 1490
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1491
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1492 1493
}

1494
NGRAPH_TEST(${BACKEND_NAME}, backwards_tan)
1495
{
1496
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1497 1498 1499 1500

    auto pi = 3.14159f;

    // Stay away from the asymptotes at 6 and 12 o'clock.
1501
    auto slop = 0.2f;
1502 1503 1504
    test::Uniform<float> rng_r(-pi / 2 + slop, pi / 2 - slop);
    test::Uniform<float> rng_l(pi / 2 + slop, (3 * pi) / 2 - slop);

1505
    Shape shape{2, 3};
1506 1507

    auto make_graph = [shape]() {
1508
        auto X = make_shared<op::Parameter>(element::f32, shape);
1509 1510
        return make_shared<Function>(make_shared<op::Tan>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1511 1512
    };

1513 1514
    auto f = make_graph();
    auto g = make_graph();
1515
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1516
    {
1517
        auto x_r = rng_r.initialize(backend->create_tensor<float>(shape));
1518

1519
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_r}, .01f, .01f));
1520

1521
        auto x_l = rng_l.initialize(backend->create_tensor<float>(shape));
1522

1523
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_l}, .01f, .01f));
1524 1525 1526
    }
}

1527
NGRAPH_TEST(${BACKEND_NAME}, backwards_tanh)
1528
{
1529
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1530 1531

    test::Uniform<float> rng(-10.0f, 10.0f);
1532
    Shape shape{2, 3};
1533
    auto make_graph = [shape]() {
1534
        auto X = make_shared<op::Parameter>(element::f32, shape);
1535 1536
        return make_shared<Function>(make_shared<op::Tanh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1537 1538
    };

1539 1540
    auto f = make_graph();
    auto g = make_graph();
1541
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1542
    {
1543
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1544

1545
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1546 1547 1548
    }
}

1549
NGRAPH_TEST(${BACKEND_NAME}, backwards_abc)
Scott Cyphers's avatar
Scott Cyphers committed
1550
{
1551
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1552

1553
    test::Uniform<float> rng(-1.0f, 1.0f);
1554
    Shape shape{2, 3};
1555 1556 1557
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1558 1559

    auto make_graph = [shape]() {
1560 1561 1562
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1563 1564
        return make_shared<Function>((X0 + X1) * X2,
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
Scott Cyphers's avatar
Scott Cyphers committed
1565
    };
1566

1567 1568
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1569
}
1570

1571
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_3d_02)
1572
{
1573
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1574 1575

    test::Uniform<float> rng(-1.0f, 1.0f);
1576
    Shape shape{2, 4, 5};
1577
    auto x = rng.initialize(backend->create_tensor(element::f32, shape));
1578 1579 1580 1581 1582 1583

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Reverse>(X, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1584
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1585
}
1586

1587
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4c1h4w4_kh2kw2_sh1sw1)
1588
{
1589
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1590
    Shape shape_a{4, 1, 4, 4}; // in NCHW
1591 1592 1593 1594 1595
    Shape maxpool_shape{4, 1, 3, 3};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{2, 2};
    auto window_movement_strides = Strides{1, 1};
1596
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1597
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1598
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1599 1600
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1601 1602
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1603 1604

    vector<float> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
pthoreho's avatar
pthoreho committed
1605 1606 1607
                            13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                            17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                            45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};
1608

1609
    vector<float> expected{// delta
1610 1611 1612
                           0, 8, 0, 0, 0, 0, 0, 4, 0, 8, 16, 0, 0, 0, 0,  0, 0, 4, 0, 4, 8,  0,
                           0, 0, 0, 4, 4, 0, 4, 4, 0, 4, 0,  0, 8, 0, 4,  0, 0, 0, 8, 0, 16, 0,
                           0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 4,  0, 4, 0, 16, 0, 0, 0, 0, 0};
1613 1614 1615 1616 1617 1618

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1619
    auto handle = backend->compile(df);
1620
    handle->call_with_validate({output}, {input, ep});
1621 1622 1623
    ASSERT_TRUE(read_vector<float>(output) == expected);
}

1624
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2c1h5w5_kh3kw3_sh2sw2)
1625
{
1626
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1627

1628
    Shape shape_a{1, 2, 5, 5}; // in NCHW
1629 1630 1631 1632 1633
    Shape maxpool_shape{1, 2, 2, 2};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
1634
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1635
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1636

1637
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1638 1639
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1640 1641
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1642 1643

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
pthoreho's avatar
pthoreho committed
1644 1645
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};
1646

1647
    vector<float> expected{// delta
1648 1649 1650
                           4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0,
                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0,
                           0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0,  4, 4, 0};
1651 1652 1653 1654 1655 1656

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1657
    auto handle = backend->compile(df);
1658
    handle->call_with_validate({output}, {input, ep});
1659
    ASSERT_TRUE(read_vector<float>(output) == expected);
1660
}
1661

1662
NGRAPH_TEST(${BACKEND_NAME}, backwards_batch_norm_training)
1663
{
1664
    const Shape input_shape{10, 4, 5, 5};
1665 1666
    const Shape channel_shape{input_shape.at(1)};
    const double eps = 1e-3;
1667

1668 1669
    // Need to keep the output elements for mean and variance from going out of scope
    // and getting freed.
1670 1671
    NodeVector goes;

1672 1673
    auto make_graph = [&input_shape, &channel_shape, &eps, &goes] {
        const element::Type& et = element::f32;
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
        auto input = make_shared<op::Parameter>(et, input_shape);
        auto gamma = make_shared<op::Parameter>(et, channel_shape);
        auto beta = make_shared<op::Parameter>(et, channel_shape);
        auto BN = make_shared<op::BatchNormTraining>(input, gamma, beta, eps);
        auto normed_input = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 0));
        auto mean = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 1));
        auto variance = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 2));
        goes.push_back(mean);
        goes.push_back(variance);
        // TODO autodiff testing with more than one result
1684 1685
        auto f =
            make_shared<Function>(ResultVector{normed_input}, ParameterVector{input, gamma, beta});
1686 1687 1688 1689
        return f;
    };

    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1690
    using T = float;
1691
    test::Uniform<T> rng(-5.0, 2.0);
1692 1693 1694
    auto input = rng.initialize(backend->create_tensor<T>(input_shape));
    auto gamma = rng.initialize(backend->create_tensor<T>(channel_shape));
    auto beta = rng.initialize(backend->create_tensor<T>(channel_shape));
1695

1696
    EXPECT_TRUE(
1697
        autodiff_numeric_compare<T>(backend.get(), make_graph, {input, gamma, beta}, .005, .005));
1698 1699
}

1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n3_c2_h3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{3, 2, 3};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{2};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 1;
    size_t sequence_axis = 0;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1713
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1714

1715 1716 1717 1718 1719
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1720

1721
    // input values don't matter
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
    vector<int> va(shape_size(shape), 0);

    vector<int> vc{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
    vector<int> expected{13, 14, 15, 16, 17, 18, 7, 8, 9, 10, 11, 12, 1, 2, 3, 4, 5, 6};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{3, 3};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1735
    auto handle = backend->compile(df);
1736
    handle->call_with_validate({da, db}, {a, b, c});
1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
    ASSERT_EQ(read_vector<int>(da), expected);
}

NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n4d2c3h2w2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{4, 2, 3, 2, 2};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{4};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 0;
    size_t sequence_axis = 2;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1753
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1754

1755 1756 1757 1758 1759
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1760

1761
    // input values don't matter
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
    vector<int> va(shape_size(shape), 0);

    std::vector<int> vc{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                        16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
                        32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
                        48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                        64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
                        80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95};

    std::vector<int> expected{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                              16, 17, 18, 19, 20, 21, 22, 23, 28, 29, 30, 31, 24, 25, 26, 27,
                              32, 33, 34, 35, 40, 41, 42, 43, 36, 37, 38, 39, 44, 45, 46, 47,
                              48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                              64, 65, 66, 67, 68, 69, 70, 71, 76, 77, 78, 79, 72, 73, 74, 75,
                              80, 81, 82, 83, 88, 89, 90, 91, 84, 85, 86, 87, 92, 93, 94, 95};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{1, 2, 1, 2};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1786
    auto handle = backend->compile(df);
1787
    handle->call_with_validate({da, db}, {a, b, c});
1788 1789
    ASSERT_EQ(read_vector<int>(da), expected);
}
1790 1791 1792 1793 1794 1795

// clang-format off
#ifdef AUTODIFF_BACKEND_${BACKEND_NAME}
#undef AUTODIFF_BACKEND_${BACKEND_NAME}
#endif
// clang-format on