autodiff.in.cpp 72.1 KB
Newer Older
1
//*****************************************************************************
2
// Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
16 17

#include <algorithm>
18
#include <functional>
19
#include <memory>
20
#include <tuple>
21 22 23

#include "gtest/gtest.h"

24 25 26 27
// clang-format off
#define AUTODIFF_BACKEND_${BACKEND_NAME}
// clang-format on

28
#include "ngraph/ngraph.hpp"
29
#include "ngraph/pass/manager.hpp"
30
#include "ngraph/runtime/reference/avg_pool.hpp"
31
#include "util/autodiff/backprop_function.hpp"
adstraw's avatar
adstraw committed
32
#include "util/autodiff/numeric_compare.hpp"
33
#include "util/random.hpp"
34
#include "util/test_control.hpp"
35 36 37
using namespace std;
using namespace ngraph;

38 39 40
static string s_manifest = "${MANIFEST}";

NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4_c1_hw4_2x2_max)
41
{
42
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
43

44
    Shape shape_a{1, 4, 4, 4}; // in CHWN
45
    Shape maxpool_shape{1, 4, 3, 3};
46 47 48

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
49
        A, AxisVector{0, 3, 1, 2}, Shape{1, 4, 4, 4}); // convert CHWN to CNHW
50
    Shape window_shape{2, 2};
51 52
    auto window_movement_strides = Strides{1, 1};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
53
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
54

55
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
56 57
    vector<int> dataEp(shape_size(maxpool_shape), 4);

58 59
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
60 61 62 63 64 65

    vector<int> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
                          13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                          17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                          45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};

66
    vector<int> expected{// delta
67 68 69 70 71 72 73 74 75
                         0, 4, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 4,  4, 12, 0,
                         0, 0, 0, 8, 0, 0, 4, 8, 0, 8, 0, 0, 8, 0, 0, 0, 0, 4, 16, 4, 16, 8,
                         0, 0, 0, 4, 0, 4, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
76
    auto handle = backend->compile(df);
77
    handle->call_with_validate({output}, {input, ep});
78
    ASSERT_TRUE(read_vector<int>(output) == expected);
79 80
}

81
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max)
82
{
83
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
84

85
    Shape shape_a{1, 5, 5, 2}; // in CHWN
86
    Shape maxpool_shape{1, 2, 2, 2};
87 88 89

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
90
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
91
    Shape window_shape{3, 3};
92 93
    auto window_movement_strides = Strides{2, 2};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
94
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
95

96
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
97 98
    vector<int> dataEp(shape_size(maxpool_shape), 4);

99 100
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
101 102 103 104 105

    vector<int> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                          30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                          19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

106
    vector<int> expected{// delta
107 108 109 110 111 112 113 114
                         4, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 4, 4, 0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
115
    auto handle = backend->compile(df);
116
    handle->call_with_validate({output}, {input, ep});
117
    ASSERT_TRUE(read_vector<int>(output) == expected);
118 119
}

120 121 122 123
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max_pad1x2_2x3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

124
    Shape shape_a{1, 5, 5, 2}; // in CHWN
125 126 127 128
    Shape maxpool_shape{1, 2, 4, 5};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    auto reshape = make_shared<op::Reshape>(
129
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
130 131 132 133 134 135
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
    Shape pad_below{1, 2};
    Shape pad_above{3, 4};
    auto maxpool = make_shared<op::MaxPool>(
        reshape, window_shape, window_movement_strides, pad_below, pad_above);
136
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
137

138
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
139 140
    vector<float> dataEp(shape_size(maxpool_shape), 4);

141 142
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
143 144 145 146 147

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

148
    vector<float> expected{// delta
149 150 151 152 153 154 155 156
                           8, 0, 0, 0, 0, 4,  0, 0, 8, 0, 0, 8, 4, 8, 0, 0, 0,
                           0, 0, 4, 4, 0, 0,  0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                           0, 0, 0, 0, 4, 12, 4, 8, 4, 0, 0, 0, 0, 4, 8, 0};
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
157
    auto handle = backend->compile(df);
158
    handle->call_with_validate({output}, {input, ep});
159 160 161
    EXPECT_EQ(expected, read_vector<float>(output));
}

162
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw2x2)
163
{
164
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
165

166
    Shape padding{1, 1};
167

168 169
    Shape shape_a{1, 1, 2, 2};
    Shape avgpool_shape{1, 1, 2, 2};
170 171

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
172
    Shape window_shape{2, 2};
173 174
    auto window_movement_strides = Strides{2, 2};
    auto avgpool =
175
        make_shared<op::AvgPool>(A, window_shape, window_movement_strides, padding, padding, false);
176
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
177

178
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
179 180
    vector<int> dataEp(shape_size(avgpool_shape), 4);

181
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
182

183
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
184 185 186 187 188 189 190 191 192 193

    vector<int> dataInput{4, 8, 12, 16};

    vector<int> expected{1, 2, 3, 4};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
194
    auto handle = backend->compile(df);
195
    handle->call_with_validate({output}, {input, ep});
196 197 198
    ASSERT_TRUE(read_vector<int>(output) == dataEp);
}

199
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw4x4)
200
{
201
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
202

203 204
    Shape shape_a{1, 1, 4, 4};
    Shape avgpool_shape{1, 1, 3, 3};
205 206

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
207
    Shape window_shape{2, 2};
208 209
    auto window_movement_strides = Strides{1, 1};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
210
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
211

212
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
213 214
    vector<int> dataEp(shape_size(avgpool_shape), 4);

215
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
216

217
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
218 219 220 221 222 223 224 225 226 227

    vector<int> dataInput{1, 3, 1, 3, 1, 3, 1, 3, 3, 5, 3, 5, 3, 5, 3, 5};

    vector<int> expected{1, 2, 2, 1, 2, 4, 4, 2, 2, 4, 4, 2, 1, 2, 2, 1};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
228
    auto handle = backend->compile(df);
229
    handle->call_with_validate({output}, {input, ep});
230 231 232
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

233
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4)
234
{
235
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
236

237 238
    Shape shape_a{2, 2, 4, 4};
    Shape avgpool_shape{2, 2, 2, 2};
239 240

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
241
    Shape window_shape{2, 2};
242 243
    auto window_movement_strides = Strides{2, 2};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
244
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
245

246
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
247 248
    vector<int> dataEp(shape_size(avgpool_shape), 12);

249
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
250

251
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
252

253
    vector<int> dataInput{// i1c1
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
                          1,
                          2,
                          6,
                          7,
                          3,
                          4,
                          4,
                          3,
                          19,
                          1,
                          2,
                          3,
                          18,
                          2,
                          3,
                          2,
270
                          // i1c2
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
                          4,
                          1,
                          5,
                          5,
                          1,
                          4,
                          5,
                          5,
                          12,
                          8,
                          2,
                          3,
                          15,
                          5,
                          3,
                          2,
287
                          // i2c1
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
                          2,
                          3,
                          7,
                          7,
                          3,
                          2,
                          3,
                          3,
                          13,
                          7,
                          1,
                          2,
                          7,
                          13,
                          3,
                          4,
304
                          // i2c2
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
                          1,
                          1,
                          2,
                          2,
                          7,
                          1,
                          2,
                          14,
                          6,
                          16,
                          4,
                          1,
                          14,
                          4,
                          4,
                          1};

    vector<int> expected(shape_size(shape_a), 3);
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
328
    auto handle = backend->compile(df);
329
    handle->call_with_validate({output}, {input, ep});
330 331 332
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

333
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_numeric)
334
{
335
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
336
    Shape shape_a{2, 2, 4, 4};
337 338 339 340
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
341
        Shape window_shape{2, 2};
342 343
        auto window_movement_strides = Strides{2, 2};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
344
        return make_shared<Function>(avgpool, ParameterVector{A});
345 346 347

    };

348 349
    auto f = make_graph();
    auto g = make_graph();
350
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
351
    {
352
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
353
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
354 355 356
    }
}

357
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_win_2x2_str_1x1_numeric)
358
{
359
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
360
    Shape shape_a{2, 2, 4, 4};
361 362 363 364
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
365
        Shape window_shape{2, 2};
366 367
        auto window_movement_strides = Strides{1, 1};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
368
        return make_shared<Function>(avgpool, ParameterVector{A});
369 370 371

    };

372 373
    auto f = make_graph();
    auto g = make_graph();
374
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
375
    {
376
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
377
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
378 379 380
    }
}

381
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw2x2_win_2x2_str_1x1_padding_numeric)
382
{
383
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
384
    Shape shape_a{2, 2, 4, 4};
385 386 387 388
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
389 390
        Shape window_shape{2, 2};
        Shape padding{1, 1};
391
        auto window_movement_strides = Strides{2, 2};
392 393
        auto avgpool = make_shared<op::AvgPool>(
            A, window_shape, window_movement_strides, padding, padding, false);
394
        return make_shared<Function>(avgpool, ParameterVector{A});
395 396 397

    };

398 399
    auto f = make_graph();
    auto g = make_graph();
400
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
401
    {
402
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
403
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
404 405 406
    }
}

407
NGRAPH_TEST(${BACKEND_NAME}, backwards_abs)
408
{
409
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
410 411 412 413 414

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
415
    Shape shape{2, 3};
416 417

    auto make_graph = [shape]() {
418
        auto X = make_shared<op::Parameter>(element::f32, shape);
419 420
        return make_shared<Function>(make_shared<op::Abs>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
421 422
    };

423 424
    auto f = make_graph();
    auto g = make_graph();
425
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
426
    {
427
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
428

429
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
430

431
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
432

433
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
434
    }
435 436
}

437
NGRAPH_TEST(${BACKEND_NAME}, backwards_acos)
438 439 440 441 442 443 444 445 446 447 448 449
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Acos>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
450
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
451 452
}

453
NGRAPH_TEST(${BACKEND_NAME}, backwards_add)
454
{
455
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
456

457
    test::Uniform<float> rng(-1.0f, 1.0f);
458
    Shape shape{2, 3};
459 460
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
461

462
    auto make_graph = [shape]() {
463 464
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
465
        return make_shared<Function>(X0 + X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
466
    };
467
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
468 469
}

470
NGRAPH_TEST(${BACKEND_NAME}, backwards_add_nested)
471
{
472
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
473

474
    test::Uniform<float> rng(-1.0f, 1.0f);
475
    Shape shape{2, 3};
476 477
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
478 479

    auto make_graph = [shape]() {
480 481
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
482 483
        return make_shared<Function>((X0 + X1) + (X1 + X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
484
    };
485
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
486 487
}

488
NGRAPH_TEST(${BACKEND_NAME}, backwards_asin)
489 490 491 492 493 494 495 496 497 498 499 500
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Asin>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
501
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
502 503
}

504
NGRAPH_TEST(${BACKEND_NAME}, backwards_atan)
505 506 507 508 509 510 511 512 513 514 515 516
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-10.0f, 10.0f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Atan>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
517
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
518 519
}

520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
NGRAPH_TEST(${BACKEND_NAME}, backwards_atan2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
    Shape shape{30};

    test::Uniform<float> rng(-5.0f, 5.0f);
    auto y = rng.initialize(backend->create_tensor<float>(shape));
    auto x = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        auto Y = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Atan2>(Y, X), ParameterVector{Y, X});
    };
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {y, x}, .01f, .01f));
}

537
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast0)
538
{
539
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
540

541
    test::Uniform<float> rng(-1.0f, 1.0f);
542
    Shape shape{3};
543
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
544 545

    auto make_graph = [shape]() {
546
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
547 548 549
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{2, 3}, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
550
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
551 552
}

553
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast1)
554
{
555
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
556

557
    test::Uniform<float> rng(-1.0f, 1.0f);
558
    Shape shape{3};
559
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
560 561

    auto make_graph = [shape]() {
562
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
563 564 565
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{3, 2}, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
566
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
567 568
}

569
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_vector)
570
{
571
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
572 573

    test::Uniform<float> rng(-1.0f, 1.0f);
574
    Shape shape_0{3};
575
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
576
    Shape shape_1{2};
577
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
578
    Shape shape_2{1};
579
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
580 581

    auto make_graph = [shape_0, shape_1, shape_2]() {
582 583 584
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
585
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
586 587
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
588 589
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
590 591
}

592
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_0)
593
{
594
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
595 596

    test::Uniform<float> rng(-1.0f, 1.0f);
597
    Shape shape_0{3, 2};
598
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
599
    Shape shape_1{2, 2};
600
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
601
    Shape shape_2{1, 2};
602
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
603 604

    auto make_graph = [shape_0, shape_1, shape_2]() {
605 606 607
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
608
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
609 610
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
611 612
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
613 614
}

615
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_1)
616
{
617
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
618 619

    test::Uniform<float> rng(-1.0f, 1.0f);
620
    Shape shape_0{2, 3};
621
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
622
    Shape shape_1{2, 2};
623
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
624
    Shape shape_2{2, 1};
625
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
626 627

    auto make_graph = [shape_0, shape_1, shape_2]() {
628 629 630
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
631
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 1),
632 633
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
634 635
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
636 637
}

638
NGRAPH_TEST(${BACKEND_NAME}, backwards_ceiling)
639
{
640
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
641

642 643
    // The numeric derivative and the symbolic one may disagree near integers, so we will dance
    // around them.
644 645 646
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
647
    Shape shape{2, 3};
648 649

    auto make_graph = [shape]() {
650
        auto X = make_shared<op::Parameter>(element::f32, shape);
651 652
        return make_shared<Function>(make_shared<op::Ceiling>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
653 654
    };

655 656
    auto f = make_graph();
    auto g = make_graph();
657
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
658
    {
659
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
660

661
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
662

663
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
664

665
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
666

667
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
668

669
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
670 671 672
    }
}

673
NGRAPH_TEST(${BACKEND_NAME}, backwards_cos)
674
{
675
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
676 677

    test::Uniform<float> rng(-10.0f, 10.0f);
678
    Shape shape{2, 3};
679
    auto make_graph = [shape]() {
680
        auto X = make_shared<op::Parameter>(element::f32, shape);
681 682
        return make_shared<Function>(make_shared<op::Cos>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
683 684
    };

685 686
    auto f = make_graph();
    auto g = make_graph();
687
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
688
    {
689
        auto x = rng.initialize(backend->create_tensor<float>(shape));
690

691
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
692 693 694
    }
}

695
NGRAPH_TEST(${BACKEND_NAME}, backwards_cosh)
696
{
697
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
698 699

    test::Uniform<float> rng(-10.0f, 10.0f);
700
    Shape shape{2, 3};
701
    auto make_graph = [shape]() {
702
        auto X = make_shared<op::Parameter>(element::f32, shape);
703 704
        return make_shared<Function>(make_shared<op::Cosh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
705 706
    };

707 708
    auto f = make_graph();
    auto g = make_graph();
709
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
710
    {
711
        auto x = rng.initialize(backend->create_tensor<float>(shape));
712

713
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
714 715 716
    }
}

717
NGRAPH_TEST(${BACKEND_NAME}, backwards_divide)
718
{
719
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
720

721 722 723
    test::Uniform<float> rng(-1.0f, 1.0f);
    test::Uniform<float> rng1(1.0f, 2.0f);
    test::Uniform<float> rng2(-2.0f, -1.0f);
724
    Shape shape{2, 3};
725 726 727
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng1.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng2.initialize(backend->create_tensor<float>(shape));
728

729
    auto make_graph = [shape]() {
730 731
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
732
        return make_shared<Function>(X0 / X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
733
    };
734 735
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
736
}
737

738
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_scalar)
739
{
740
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
741

742
    test::Uniform<float> rng(-1.0f, 1.0f);
743 744
    Shape shape0{};
    Shape shape1{};
745 746
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
747 748

    auto make_graph = [shape0, shape1]() {
749 750
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
751 752 753
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
754
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
755 756
}

757
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_tensor)
758
{
759
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
760

761
    test::Uniform<float> rng(-1.0f, 1.0f);
762 763
    Shape shape0{};
    Shape shape1{3, 4};
764 765
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
766 767

    auto make_graph = [shape0, shape1]() {
768 769
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
770 771 772
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
773
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
774 775
}

776
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_scalar)
777
{
778
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
779

780
    test::Uniform<float> rng(-1.0f, 1.0f);
781 782
    Shape shape0{3, 4};
    Shape shape1{};
783 784
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
785 786

    auto make_graph = [shape0, shape1]() {
787 788
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
789 790 791
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
792
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
793 794
}

795
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_vector_vector)
796
{
797
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
798

799
    test::Uniform<float> rng(-1.0f, 1.0f);
800 801
    Shape shape0{3};
    Shape shape1{3};
802 803
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
804 805

    auto make_graph = [shape0, shape1]() {
806 807
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
808 809 810
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
811
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
812 813
}

814
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_vector)
815
{
816
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
817

818
    test::Uniform<float> rng(-1.0f, 1.0f);
819 820
    Shape shape0{4, 3};
    Shape shape1{3};
821 822
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
823 824

    auto make_graph = [shape0, shape1]() {
825 826
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
827 828 829
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
830
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
831 832
}

833
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor2_tensor2)
834
{
835
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
836

837
    test::Uniform<float> rng(-1.0f, 1.0f);
838 839
    Shape shape0{4, 3};
    Shape shape1{3, 5};
840 841
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
842 843

    auto make_graph = [shape0, shape1]() {
844 845
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
846 847
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Adam Procter's avatar
Adam Procter committed
848
    };
849
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Adam Procter's avatar
Adam Procter committed
850 851
}

852
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor3_tensor3)
Adam Procter's avatar
Adam Procter committed
853
{
854
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Adam Procter's avatar
Adam Procter committed
855 856

    test::Uniform<float> rng(-1.0f, 1.0f);
857 858
    Shape shape0{2, 4, 3};
    Shape shape1{4, 3, 3};
859 860
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
Adam Procter's avatar
Adam Procter committed
861 862

    auto make_graph = [shape0, shape1]() {
863 864
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
Adam Procter's avatar
Adam Procter committed
865 866
        return make_shared<Function>(make_shared<op::Dot>(X0, X1, 2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
867
    };
868
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
869 870
}

871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
NGRAPH_TEST(${BACKEND_NAME}, backwards_batchmatmul_tensor2_tensor2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape0{3, 4, 5};
    Shape shape1{3, 5, 6};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));

    auto make_graph = [shape0, shape1]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
        return make_shared<Function>(make_shared<op::BatchMatMul>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };

    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
}
889

890
NGRAPH_TEST(${BACKEND_NAME}, backwards_exp)
Scott Cyphers's avatar
Scott Cyphers committed
891
{
892
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
893

894
    test::Uniform<float> rng(-1.0f, 1.0f);
895
    Shape shape{2, 3};
896
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
897 898

    auto make_graph = [shape]() {
899
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
900 901
        return make_shared<Function>(make_shared<op::Exp>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
902
    };
903
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
904 905
}

906
NGRAPH_TEST(${BACKEND_NAME}, backwards_floor)
907
{
908
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
909

910 911
    // The numeric derivative and the symbolic one may disagree near integers, so we will dance
    // around them.
912 913 914
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
915
    Shape shape{2, 3};
916 917

    auto make_graph = [shape]() {
918
        auto X = make_shared<op::Parameter>(element::f32, shape);
919 920
        return make_shared<Function>(make_shared<op::Floor>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
921 922
    };

923 924
    auto f = make_graph();
    auto g = make_graph();
925
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
926
    {
927
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
928

929
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
930

931
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
932

933
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
934

935
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
936

937
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
938 939 940
    }
}

941
NGRAPH_TEST(${BACKEND_NAME}, backwards_log)
Scott Cyphers's avatar
Scott Cyphers committed
942
{
943
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
944

945
    test::Uniform<float> rng(1.0f, 2.0f);
946
    Shape shape{2, 3};
947
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
948 949

    auto make_graph = [shape]() {
950
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
951 952
        return make_shared<Function>(make_shared<op::Log>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
953
    };
954
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
955 956
}

957
NGRAPH_TEST(${BACKEND_NAME}, backwards_maximum)
958
{
959
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
960

961
    test::Uniform<float> rng(-1.0f, 1.0f);
962
    Shape shape{2, 3};
963 964
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
965 966

    auto make_graph = [shape]() {
967 968
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
969 970 971
        return make_shared<Function>(make_shared<op::Maximum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
972
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
973 974
}

975
NGRAPH_TEST(${BACKEND_NAME}, backwards_minimum)
976
{
977
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
978

979
    test::Uniform<float> rng(-1.0f, 1.0f);
980
    Shape shape{2, 3};
981 982
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
983 984

    auto make_graph = [shape]() {
985 986
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
987 988 989
        return make_shared<Function>(make_shared<op::Minimum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
990
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
991 992
}

993
NGRAPH_TEST(${BACKEND_NAME}, backwards_multiply)
994
{
995
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
996

997
    test::Uniform<float> rng(-1.0f, 1.0f);
998
    Shape shape{2, 3};
999 1000
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
1001

1002
    auto make_graph = [shape]() {
1003 1004
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1005
        return make_shared<Function>(X0 * X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
1006
    };
1007
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1008
}
Scott Cyphers's avatar
Scott Cyphers committed
1009

1010
NGRAPH_TEST(${BACKEND_NAME}, backwards_negative)
Scott Cyphers's avatar
Scott Cyphers committed
1011
{
1012
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1013

1014
    test::Uniform<float> rng(-1.0f, 1.0f);
1015
    Shape shape{2, 3};
1016
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1017 1018

    auto make_graph = [shape]() {
1019
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1020
        return make_shared<Function>(-X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1021
    };
1022
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1023 1024
}

1025
NGRAPH_TEST(${BACKEND_NAME}, backwards_parameter)
Scott Cyphers's avatar
Scott Cyphers committed
1026
{
1027
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1028

1029
    test::Uniform<float> rng(-1.0f, 1.0f);
1030
    Shape shape{2, 3};
1031
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1032
    auto make_graph = [shape]() {
1033
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1034
        return make_shared<Function>(X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1035
    };
1036
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
1037 1038
}

1039
NGRAPH_TEST(${BACKEND_NAME}, backwards_power)
1040
{
1041
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1042 1043 1044

    test::Uniform<float> rng_neg(-5.0f, -0.5f);
    test::Uniform<float> rng_pos(0.5f, 5.0f);
1045
    Shape shape{2, 3};
1046 1047

    auto make_graph = [shape]() {
1048 1049
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1050 1051 1052 1053
        return make_shared<Function>(std::make_shared<op::Power>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };

1054 1055
    auto x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_neg.initialize(backend->create_tensor<float>(shape));
1056

1057
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1058

1059 1060
    x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1061

1062
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1063 1064
}

1065
NGRAPH_TEST(${BACKEND_NAME}, backwards_relu)
1066
{
1067
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1068 1069 1070 1071

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
1072 1073
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1074 1075 1076 1077 1078 1079 1080

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Relu>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1081 1082
    auto f = make_graph();
    auto g = make_graph();
1083 1084
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
1085
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1086

1087
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1088

1089
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1090

1091
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1092 1093 1094
    }
}

1095
NGRAPH_TEST(${BACKEND_NAME}, backwards_replace_slice)
1096
{
1097
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1098 1099

    test::Uniform<float> rng(-10.0f, 10.0f);
1100 1101
    Shape shape_x{5, 5};
    Shape shape_y{2, 2};
1102
    auto make_graph = [shape_x, shape_y]() {
1103 1104
        auto X = make_shared<op::Parameter>(element::f32, shape_x);
        auto Y = make_shared<op::Parameter>(element::f32, shape_y);
1105 1106 1107 1108 1109
        return make_shared<Function>(
            make_shared<op::ReplaceSlice>(X, Y, Coordinate{2, 3}, Coordinate{4, 5}),
            std::vector<std::shared_ptr<op::Parameter>>{X, Y});
    };

1110 1111
    auto f = make_graph();
    auto g = make_graph();
1112
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1113
    {
1114 1115
        auto x = rng.initialize(backend->create_tensor<float>(shape_x));
        auto y = rng.initialize(backend->create_tensor<float>(shape_y));
1116

1117
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x, y}, .01f, .01f));
1118 1119 1120
    }
}

1121
NGRAPH_TEST(${BACKEND_NAME}, backwards_reshape)
1122
{
1123
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1124

1125
    test::Uniform<float> rng(-1.0f, 1.0f);
1126
    Shape shape{3, 4};
1127
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
1128 1129

    auto make_graph = [shape]() {
1130
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1131 1132 1133
        return make_shared<Function>(make_shared<op::Reshape>(X0, AxisVector{1, 0}, Shape{4, 3}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1134
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1135 1136
}

1137
NGRAPH_TEST(${BACKEND_NAME}, backwards_select)
1138
{
1139
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1140 1141

    test::Uniform<float> rng(-10.0f, 10.0f);
1142
    Shape shape{2, 3};
1143
    auto make_graph = [shape]() {
1144 1145 1146
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1147 1148 1149 1150
        return make_shared<Function>(make_shared<op::Select>(X0, X1, X2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1151 1152
    auto f = make_graph();
    auto g = make_graph();
1153
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1154
    {
1155
        auto x0 = backend->create_tensor(element::boolean, shape);
1156
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1157 1158 1159 1160
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1161
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1162 1163 1164
    }
}

1165
NGRAPH_TEST(${BACKEND_NAME}, backwards_select_nested)
1166
{
1167
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1168 1169

    test::Uniform<float> rng(-10.0f, 10.0f);
1170
    Shape shape{2, 3};
1171
    auto make_graph = [shape]() {
1172 1173 1174
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1175
        return make_shared<Function>(make_shared<op::Select>(X0, X2 + X1, X2 - X1),
1176 1177 1178
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1179 1180
    auto f = make_graph();
    auto g = make_graph();
1181
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1182
    {
1183
        auto x0 = backend->create_tensor(element::boolean, shape);
1184
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1185 1186 1187 1188
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1189
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1190 1191 1192
    }
}

1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
NGRAPH_TEST(${BACKEND_NAME}, backwards_sigmoid)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Sigmoid>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1209 1210
    auto f = make_graph();
    auto g = make_graph();
1211 1212 1213 1214
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));

1215
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1216 1217 1218

        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));

1219
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1220 1221 1222
    }
}

1223
NGRAPH_TEST(${BACKEND_NAME}, backwards_sign)
1224
{
1225
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1226 1227 1228 1229 1230

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
1231
    Shape shape{2, 3};
1232 1233

    auto make_graph = [shape]() {
1234
        auto X = make_shared<op::Parameter>(element::f32, shape);
1235 1236
        return make_shared<Function>(make_shared<op::Sign>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1237 1238
    };

1239 1240
    auto f = make_graph();
    auto g = make_graph();
1241
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1242
    {
1243
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1244

1245
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1246

1247
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1248

1249
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1250 1251 1252
    }
}

1253
NGRAPH_TEST(${BACKEND_NAME}, backwards_sin)
1254
{
1255
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1256 1257

    test::Uniform<float> rng(-10.0f, 10.0f);
1258
    Shape shape{2, 3};
1259
    auto make_graph = [shape]() {
1260
        auto X = make_shared<op::Parameter>(element::f32, shape);
1261 1262
        return make_shared<Function>(make_shared<op::Sin>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1263 1264
    };

1265 1266
    auto f = make_graph();
    auto g = make_graph();
1267
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1268
    {
1269
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1270

1271
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1272 1273 1274
    }
}

1275
NGRAPH_TEST(${BACKEND_NAME}, backwards_sinh)
1276
{
1277
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1278 1279

    test::Uniform<float> rng(-10.0f, 10.0f);
1280
    Shape shape{2, 3};
1281
    auto make_graph = [shape]() {
1282
        auto X = make_shared<op::Parameter>(element::f32, shape);
1283 1284
        return make_shared<Function>(make_shared<op::Sinh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1285 1286
    };

1287 1288
    auto f = make_graph();
    auto g = make_graph();
1289
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1290
    {
1291
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1292

1293
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1294 1295 1296
    }
}

1297
NGRAPH_TEST(${BACKEND_NAME}, backwards_slice)
1298
{
1299
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1300
    test::Uniform<float> rng(-10.0f, 10.0f);
1301
    Shape shape{5, 5};
1302
    auto make_graph = [shape]() {
1303
        auto X = make_shared<op::Parameter>(element::f32, shape);
1304 1305 1306 1307
        return make_shared<Function>(make_shared<op::Slice>(X, Coordinate{2, 3}, Coordinate{4, 5}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1308 1309
    auto f = make_graph();
    auto g = make_graph();
1310
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1311
    {
1312
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1313

1314
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1315 1316 1317
    }
}

1318
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_all)
1319
{
1320
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1321

adstraw's avatar
adstraw committed
1322
    test::Uniform<float> rng(-1.0f, 1.0f);
1323
    Shape shape{2, 3};
1324
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1325

1326
    auto make_graph = [shape]() {
adstraw's avatar
adstraw committed
1327 1328 1329
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
1330
    };
1331
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1332
}
1333

1334
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_axis)
adstraw's avatar
adstraw committed
1335
{
1336
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1337

adstraw's avatar
adstraw committed
1338 1339
    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3};
1340
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1341 1342 1343 1344 1345 1346

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1347
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1348 1349
}

1350
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_underflow)
adstraw's avatar
adstraw committed
1351
{
1352
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1353 1354 1355 1356

    auto low = std::numeric_limits<float>::lowest();

    Shape shape{2, 3};
1357
    auto x0 = backend->create_tensor(element::f32, shape);
adstraw's avatar
adstraw committed
1358 1359 1360 1361 1362 1363 1364
    copy_data(x0, vector<float>{low, 1, 2, 3, 4, 5});

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1365
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1366 1367
}

1368
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_3d)
adstraw's avatar
adstraw committed
1369
{
1370
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1371 1372 1373

    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3, 4};
1374
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1375 1376 1377 1378 1379 1380

    auto make_graph0 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1381
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph0, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1382 1383 1384 1385 1386 1387

    auto make_graph1 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1388
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph1, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1389 1390 1391 1392 1393 1394

    auto make_graph2 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1395
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph2, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1396 1397 1398 1399 1400 1401

    auto make_graph01 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1402
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph01, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1403 1404 1405 1406 1407 1408

    auto make_graph02 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1409
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph02, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1410 1411 1412 1413 1414 1415

    auto make_graph12 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1416
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph12, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1417 1418 1419 1420 1421 1422

    auto make_graph012 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1423
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph012, {x0}, .01f, .01f));
1424 1425
}

1426
NGRAPH_TEST(${BACKEND_NAME}, backwards_subtract)
Scott Cyphers's avatar
Scott Cyphers committed
1427
{
1428
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1429

1430
    test::Uniform<float> rng(-1.0f, 1.0f);
1431
    Shape shape{2, 3};
1432 1433
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1434 1435

    auto make_graph = [shape]() {
1436 1437
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1438
        return make_shared<Function>(X0 - X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Scott Cyphers's avatar
Scott Cyphers committed
1439
    };
1440
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1441
}
Scott Cyphers's avatar
Scott Cyphers committed
1442

1443
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_v2s)
1444
{
1445
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1446 1447

    test::Uniform<float> rng(-1.0f, 1.0f);
1448
    Shape shape{8};
1449
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1450 1451

    auto make_graph = [shape]() {
1452
        auto X = make_shared<op::Parameter>(element::f32, shape);
1453 1454 1455
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1456
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1457 1458
}

1459
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2s)
1460
{
1461
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1462 1463

    test::Uniform<float> rng(-1.0f, 1.0f);
1464
    Shape shape{8, 9};
1465
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1466 1467

    auto make_graph = [shape]() {
1468
        auto X = make_shared<op::Parameter>(element::f32, shape);
1469 1470 1471
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1472
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1473 1474
}

1475
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_0)
1476
{
1477
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1478 1479

    test::Uniform<float> rng(-1.0f, 1.0f);
1480
    Shape shape{8, 9};
1481
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1482 1483

    auto make_graph = [shape]() {
1484
        auto X = make_shared<op::Parameter>(element::f32, shape);
1485 1486 1487
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1488
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1489 1490
}

1491
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_1)
1492
{
1493
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1494 1495

    test::Uniform<float> rng(-1.0f, 1.0f);
1496
    Shape shape{8, 9};
1497
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1498 1499

    auto make_graph = [shape]() {
1500
        auto X = make_shared<op::Parameter>(element::f32, shape);
1501 1502 1503
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1504
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1505 1506
}

1507
NGRAPH_TEST(${BACKEND_NAME}, backwards_tan)
1508
{
1509
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1510 1511 1512 1513

    auto pi = 3.14159f;

    // Stay away from the asymptotes at 6 and 12 o'clock.
1514
    auto slop = 0.2f;
1515 1516 1517
    test::Uniform<float> rng_r(-pi / 2 + slop, pi / 2 - slop);
    test::Uniform<float> rng_l(pi / 2 + slop, (3 * pi) / 2 - slop);

1518
    Shape shape{2, 3};
1519 1520

    auto make_graph = [shape]() {
1521
        auto X = make_shared<op::Parameter>(element::f32, shape);
1522 1523
        return make_shared<Function>(make_shared<op::Tan>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1524 1525
    };

1526 1527
    auto f = make_graph();
    auto g = make_graph();
1528
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1529
    {
1530
        auto x_r = rng_r.initialize(backend->create_tensor<float>(shape));
1531

1532
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_r}, .01f, .01f));
1533

1534
        auto x_l = rng_l.initialize(backend->create_tensor<float>(shape));
1535

1536
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_l}, .01f, .01f));
1537 1538 1539
    }
}

1540
NGRAPH_TEST(${BACKEND_NAME}, backwards_tanh)
1541
{
1542
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1543 1544

    test::Uniform<float> rng(-10.0f, 10.0f);
1545
    Shape shape{2, 3};
1546
    auto make_graph = [shape]() {
1547
        auto X = make_shared<op::Parameter>(element::f32, shape);
1548 1549
        return make_shared<Function>(make_shared<op::Tanh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1550 1551
    };

1552 1553
    auto f = make_graph();
    auto g = make_graph();
1554
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1555
    {
1556
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1557

1558
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1559 1560 1561
    }
}

1562
NGRAPH_TEST(${BACKEND_NAME}, backwards_abc)
Scott Cyphers's avatar
Scott Cyphers committed
1563
{
1564
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1565

1566
    test::Uniform<float> rng(-1.0f, 1.0f);
1567
    Shape shape{2, 3};
1568 1569 1570
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1571 1572

    auto make_graph = [shape]() {
1573 1574 1575
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1576 1577
        return make_shared<Function>((X0 + X1) * X2,
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
Scott Cyphers's avatar
Scott Cyphers committed
1578
    };
1579

1580 1581
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1582
}
1583

1584
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_3d_02)
1585
{
1586
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1587 1588

    test::Uniform<float> rng(-1.0f, 1.0f);
1589
    Shape shape{2, 4, 5};
1590
    auto x = rng.initialize(backend->create_tensor(element::f32, shape));
1591 1592 1593 1594 1595 1596

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Reverse>(X, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1597
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1598
}
1599

1600
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4c1h4w4_kh2kw2_sh1sw1)
1601
{
1602
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1603
    Shape shape_a{4, 1, 4, 4}; // in NCHW
1604 1605 1606 1607 1608
    Shape maxpool_shape{4, 1, 3, 3};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{2, 2};
    auto window_movement_strides = Strides{1, 1};
1609
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1610
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1611
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1612 1613
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1614 1615
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1616 1617

    vector<float> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
pthoreho's avatar
pthoreho committed
1618 1619 1620
                            13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                            17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                            45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};
1621

1622
    vector<float> expected{// delta
1623 1624 1625
                           0, 8, 0, 0, 0, 0, 0, 4, 0, 8, 16, 0, 0, 0, 0,  0, 0, 4, 0, 4, 8,  0,
                           0, 0, 0, 4, 4, 0, 4, 4, 0, 4, 0,  0, 8, 0, 4,  0, 0, 0, 8, 0, 16, 0,
                           0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 4,  0, 4, 0, 16, 0, 0, 0, 0, 0};
1626 1627 1628 1629 1630 1631

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1632
    auto handle = backend->compile(df);
1633
    handle->call_with_validate({output}, {input, ep});
1634 1635 1636
    ASSERT_TRUE(read_vector<float>(output) == expected);
}

1637
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2c1h5w5_kh3kw3_sh2sw2)
1638
{
1639
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1640

1641
    Shape shape_a{1, 2, 5, 5}; // in NCHW
1642 1643 1644 1645 1646
    Shape maxpool_shape{1, 2, 2, 2};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
1647
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1648
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1649

1650
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1651 1652
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1653 1654
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1655 1656

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
pthoreho's avatar
pthoreho committed
1657 1658
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};
1659

1660
    vector<float> expected{// delta
1661 1662 1663
                           4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0,
                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0,
                           0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0,  4, 4, 0};
1664 1665 1666 1667 1668 1669

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1670
    auto handle = backend->compile(df);
1671
    handle->call_with_validate({output}, {input, ep});
1672
    ASSERT_TRUE(read_vector<float>(output) == expected);
1673
}
1674

1675
NGRAPH_TEST(${BACKEND_NAME}, backwards_batch_norm_training_4d)
1676
{
1677
    const Shape input_shape{10, 4, 5, 5};
1678 1679
    const Shape channel_shape{input_shape.at(1)};
    const double eps = 1e-3;
1680

1681 1682
    // Need to keep the output elements for mean and variance from going out of scope
    // and getting freed.
1683 1684
    NodeVector goes;

1685 1686
    auto make_graph = [&input_shape, &channel_shape, &eps, &goes] {
        const element::Type& et = element::f32;
1687 1688 1689 1690 1691
        auto input = make_shared<op::Parameter>(et, input_shape);
        auto gamma = make_shared<op::Parameter>(et, channel_shape);
        auto beta = make_shared<op::Parameter>(et, channel_shape);
        auto BN = make_shared<op::BatchNormTraining>(input, gamma, beta, eps);
        auto normed_input = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 0));
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
        auto mean = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 1));
        auto variance = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 2));
        goes.push_back(mean);
        goes.push_back(variance);
        // TODO autodiff testing with more than one result
        auto f =
            make_shared<Function>(ResultVector{normed_input}, ParameterVector{input, gamma, beta});
        return f;
    };

    auto backend = runtime::Backend::create("${BACKEND_NAME}");
    using T = float;
    test::Uniform<T> rng(-5.0, 2.0);
    auto input = rng.initialize(backend->create_tensor<T>(input_shape));
    auto gamma = rng.initialize(backend->create_tensor<T>(channel_shape));
    auto beta = rng.initialize(backend->create_tensor<T>(channel_shape));

    EXPECT_TRUE(
        autodiff_numeric_compare<T>(backend.get(), make_graph, {input, gamma, beta}, .005, .005));
}

NGRAPH_TEST(${BACKEND_NAME}, backwards_batch_norm_training_3d)
{
    const Shape input_shape{10, 4, 5};
    const Shape channel_shape{input_shape.at(1)};
    const double eps = 1e-3;

    // Need to keep the output elements for mean and variance from going out of scope
    // and getting freed.
    NodeVector goes;

    auto make_graph = [&input_shape, &channel_shape, &eps, &goes] {
        const element::Type& et = element::f32;
        auto input = make_shared<op::Parameter>(et, input_shape);
        auto gamma = make_shared<op::Parameter>(et, channel_shape);
        auto beta = make_shared<op::Parameter>(et, channel_shape);
        auto BN = make_shared<op::BatchNormTraining>(input, gamma, beta, eps);
        auto normed_input = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 0));
1730 1731 1732 1733 1734
        auto mean = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 1));
        auto variance = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 2));
        goes.push_back(mean);
        goes.push_back(variance);
        // TODO autodiff testing with more than one result
1735 1736
        auto f =
            make_shared<Function>(ResultVector{normed_input}, ParameterVector{input, gamma, beta});
1737 1738 1739 1740
        return f;
    };

    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1741
    using T = float;
1742
    test::Uniform<T> rng(-5.0, 2.0);
1743 1744 1745
    auto input = rng.initialize(backend->create_tensor<T>(input_shape));
    auto gamma = rng.initialize(backend->create_tensor<T>(channel_shape));
    auto beta = rng.initialize(backend->create_tensor<T>(channel_shape));
1746

1747
    EXPECT_TRUE(
1748
        autodiff_numeric_compare<T>(backend.get(), make_graph, {input, gamma, beta}, .005, .005));
1749 1750
}

1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n3_c2_h3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{3, 2, 3};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{2};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 1;
    size_t sequence_axis = 0;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1764
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1765

1766 1767 1768 1769 1770
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1771

1772
    // input values don't matter
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
    vector<int> va(shape_size(shape), 0);

    vector<int> vc{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
    vector<int> expected{13, 14, 15, 16, 17, 18, 7, 8, 9, 10, 11, 12, 1, 2, 3, 4, 5, 6};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{3, 3};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1786
    auto handle = backend->compile(df);
1787
    handle->call_with_validate({da, db}, {a, b, c});
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
    ASSERT_EQ(read_vector<int>(da), expected);
}

NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n4d2c3h2w2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{4, 2, 3, 2, 2};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{4};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 0;
    size_t sequence_axis = 2;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1804
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1805

1806 1807 1808 1809 1810
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1811

1812
    // input values don't matter
1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
    vector<int> va(shape_size(shape), 0);

    std::vector<int> vc{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                        16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
                        32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
                        48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                        64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
                        80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95};

    std::vector<int> expected{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                              16, 17, 18, 19, 20, 21, 22, 23, 28, 29, 30, 31, 24, 25, 26, 27,
                              32, 33, 34, 35, 40, 41, 42, 43, 36, 37, 38, 39, 44, 45, 46, 47,
                              48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                              64, 65, 66, 67, 68, 69, 70, 71, 76, 77, 78, 79, 72, 73, 74, 75,
                              80, 81, 82, 83, 88, 89, 90, 91, 84, 85, 86, 87, 92, 93, 94, 95};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{1, 2, 1, 2};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1837
    auto handle = backend->compile(df);
1838
    handle->call_with_validate({da, db}, {a, b, c});
1839 1840
    ASSERT_EQ(read_vector<int>(da), expected);
}
1841 1842 1843 1844 1845 1846

// clang-format off
#ifdef AUTODIFF_BACKEND_${BACKEND_NAME}
#undef AUTODIFF_BACKEND_${BACKEND_NAME}
#endif
// clang-format on