autodiff.in.cpp 70 KB
Newer Older
1
//*****************************************************************************
2
// Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
16 17

#include <algorithm>
18
#include <functional>
19
#include <memory>
20
#include <tuple>
21 22 23

#include "gtest/gtest.h"

24 25 26 27
// clang-format off
#define AUTODIFF_BACKEND_${BACKEND_NAME}
// clang-format on

28
#include "ngraph/ngraph.hpp"
29 30
#include "ngraph/pass/manager.hpp"
#include "ngraph/runtime/cpu/pass/cpu_mat_fusion.hpp"
31
#include "ngraph/runtime/reference/avg_pool.hpp"
32
#include "util/autodiff/backprop_function.hpp"
adstraw's avatar
adstraw committed
33
#include "util/autodiff/numeric_compare.hpp"
34
#include "util/random.hpp"
35
#include "util/test_control.hpp"
36 37 38
using namespace std;
using namespace ngraph;

39 40 41
static string s_manifest = "${MANIFEST}";

NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4_c1_hw4_2x2_max)
42
{
43
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
44

45
    Shape shape_a{1, 4, 4, 4}; // in CHWN
46
    Shape maxpool_shape{1, 4, 3, 3};
47 48 49

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
50
        A, AxisVector{0, 3, 1, 2}, Shape{1, 4, 4, 4}); // convert CHWN to CNHW
51
    Shape window_shape{2, 2};
52 53
    auto window_movement_strides = Strides{1, 1};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
54
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
55

56
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
57 58
    vector<int> dataEp(shape_size(maxpool_shape), 4);

59 60
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
61 62 63 64 65 66

    vector<int> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
                          13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                          17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                          45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};

67
    vector<int> expected{// delta
68 69 70 71 72 73 74 75 76
                         0, 4, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 4,  4, 12, 0,
                         0, 0, 0, 8, 0, 0, 4, 8, 0, 8, 0, 0, 8, 0, 0, 0, 0, 4, 16, 4, 16, 8,
                         0, 0, 0, 4, 0, 4, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
77 78
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
79
    ASSERT_TRUE(read_vector<int>(output) == expected);
80 81
}

82
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max)
83
{
84
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
85

86
    Shape shape_a{1, 5, 5, 2}; // in CHWN
87
    Shape maxpool_shape{1, 2, 2, 2};
88 89 90

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
    auto reshape = make_shared<op::Reshape>(
91
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
92
    Shape window_shape{3, 3};
93 94
    auto window_movement_strides = Strides{2, 2};
    auto maxpool = make_shared<op::MaxPool>(reshape, window_shape, window_movement_strides);
95
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
96

97
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, maxpool_shape);
98 99
    vector<int> dataEp(shape_size(maxpool_shape), 4);

100 101
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
102 103 104 105 106

    vector<int> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                          30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                          19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

107
    vector<int> expected{// delta
108 109 110 111 112 113 114 115
                         4, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 4, 4, 0};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
116 117
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
118
    ASSERT_TRUE(read_vector<int>(output) == expected);
119 120
}

121 122 123 124
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2_c1_hw5_3x3_str2_max_pad1x2_2x3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

125
    Shape shape_a{1, 5, 5, 2}; // in CHWN
126 127 128 129
    Shape maxpool_shape{1, 2, 4, 5};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    auto reshape = make_shared<op::Reshape>(
130
        A, AxisVector{0, 3, 1, 2}, Shape{1, 2, 5, 5}); // convert CHWN to CNHW
131 132 133 134 135 136
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
    Shape pad_below{1, 2};
    Shape pad_above{3, 4};
    auto maxpool = make_shared<op::MaxPool>(
        reshape, window_shape, window_movement_strides, pad_below, pad_above);
137
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
138

139
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
140 141
    vector<float> dataEp(shape_size(maxpool_shape), 4);

142 143
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
144 145 146 147 148

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};

149
    vector<float> expected{// delta
150 151 152 153 154 155 156 157
                           8, 0, 0, 0, 0, 4,  0, 0, 8, 0, 0, 8, 4, 8, 0, 0, 0,
                           0, 0, 4, 4, 0, 0,  0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                           0, 0, 0, 0, 4, 12, 4, 8, 4, 0, 0, 0, 0, 4, 8, 0};
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
158 159
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
160 161 162
    EXPECT_EQ(expected, read_vector<float>(output));
}

163
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw2x2)
164
{
165
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
166

167
    Shape padding{1, 1};
168

169 170
    Shape shape_a{1, 1, 2, 2};
    Shape avgpool_shape{1, 1, 2, 2};
171 172

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
173
    Shape window_shape{2, 2};
174 175
    auto window_movement_strides = Strides{2, 2};
    auto avgpool =
176
        make_shared<op::AvgPool>(A, window_shape, window_movement_strides, padding, padding, false);
177
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
178

179
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
180 181
    vector<int> dataEp(shape_size(avgpool_shape), 4);

182
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
183

184
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
185 186 187 188 189 190 191 192 193 194

    vector<int> dataInput{4, 8, 12, 16};

    vector<int> expected{1, 2, 3, 4};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
195 196
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
197 198 199
    ASSERT_TRUE(read_vector<int>(output) == dataEp);
}

200
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n1_c1_hw4x4)
201
{
202
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
203

204 205
    Shape shape_a{1, 1, 4, 4};
    Shape avgpool_shape{1, 1, 3, 3};
206 207

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
208
    Shape window_shape{2, 2};
209 210
    auto window_movement_strides = Strides{1, 1};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
211
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
212

213
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
214 215
    vector<int> dataEp(shape_size(avgpool_shape), 4);

216
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
217

218
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
219 220 221 222 223 224 225 226 227 228

    vector<int> dataInput{1, 3, 1, 3, 1, 3, 1, 3, 3, 5, 3, 5, 3, 5, 3, 5};

    vector<int> expected{1, 2, 2, 1, 2, 4, 4, 2, 2, 4, 4, 2, 1, 2, 2, 1};

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
229 230
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
231 232 233
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

234
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4)
235
{
236
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
237

238 239
    Shape shape_a{2, 2, 4, 4};
    Shape avgpool_shape{2, 2, 2, 2};
240 241

    auto A = make_shared<op::Parameter>(element::i32, shape_a);
242
    Shape window_shape{2, 2};
243 244
    auto window_movement_strides = Strides{2, 2};
    auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
245
    auto f = make_shared<Function>(avgpool, ParameterVector{A});
246

247
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::i32, avgpool_shape);
248 249
    vector<int> dataEp(shape_size(avgpool_shape), 12);

250
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::i32, shape_a);
251

252
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::i32, shape_a);
253

254
    vector<int> dataInput{// i1c1
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
                          1,
                          2,
                          6,
                          7,
                          3,
                          4,
                          4,
                          3,
                          19,
                          1,
                          2,
                          3,
                          18,
                          2,
                          3,
                          2,
271
                          // i1c2
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
                          4,
                          1,
                          5,
                          5,
                          1,
                          4,
                          5,
                          5,
                          12,
                          8,
                          2,
                          3,
                          15,
                          5,
                          3,
                          2,
288
                          // i2c1
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
                          2,
                          3,
                          7,
                          7,
                          3,
                          2,
                          3,
                          3,
                          13,
                          7,
                          1,
                          2,
                          7,
                          13,
                          3,
                          4,
305
                          // i2c2
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
                          1,
                          1,
                          2,
                          2,
                          7,
                          1,
                          2,
                          14,
                          6,
                          16,
                          4,
                          1,
                          14,
                          4,
                          4,
                          1};

    vector<int> expected(shape_size(shape_a), 3);
    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::i32, avgpool_shape);
    auto df = autodiff::backprop_function(f);
329 330
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
331 332 333
    ASSERT_TRUE(read_vector<int>(output) == expected);
}

334
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_numeric)
335
{
336
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
337
    Shape shape_a{2, 2, 4, 4};
338 339 340 341
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
342
        Shape window_shape{2, 2};
343 344
        auto window_movement_strides = Strides{2, 2};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
345
        return make_shared<Function>(avgpool, ParameterVector{A});
346 347 348

    };

349 350
    auto f = make_graph();
    auto g = make_graph();
351
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
352
    {
353
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
354
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
355 356 357
    }
}

358
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw4x4_win_2x2_str_1x1_numeric)
359
{
360
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
361
    Shape shape_a{2, 2, 4, 4};
362 363 364 365
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
366
        Shape window_shape{2, 2};
367 368
        auto window_movement_strides = Strides{1, 1};
        auto avgpool = make_shared<op::AvgPool>(A, window_shape, window_movement_strides);
369
        return make_shared<Function>(avgpool, ParameterVector{A});
370 371 372

    };

373 374
    auto f = make_graph();
    auto g = make_graph();
375
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
376
    {
377
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
378
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
379 380 381
    }
}

382
NGRAPH_TEST(${BACKEND_NAME}, backwards_avgpool_n2_c2_hw2x2_win_2x2_str_1x1_padding_numeric)
383
{
384
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
385
    Shape shape_a{2, 2, 4, 4};
386 387 388 389
    test::Uniform<float> rng(1.0f, 10.0f);

    auto make_graph = [shape_a]() {
        auto A = make_shared<op::Parameter>(element::f32, shape_a);
390 391
        Shape window_shape{2, 2};
        Shape padding{1, 1};
392
        auto window_movement_strides = Strides{2, 2};
393 394
        auto avgpool = make_shared<op::AvgPool>(
            A, window_shape, window_movement_strides, padding, padding, false);
395
        return make_shared<Function>(avgpool, ParameterVector{A});
396 397 398

    };

399 400
    auto f = make_graph();
    auto g = make_graph();
401
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
402
    {
403
        auto x = rng.initialize(backend->create_tensor(element::f32, shape_a));
404
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
405 406 407
    }
}

408
NGRAPH_TEST(${BACKEND_NAME}, backwards_abs)
409
{
410
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
411 412 413 414 415

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
416
    Shape shape{2, 3};
417 418

    auto make_graph = [shape]() {
419
        auto X = make_shared<op::Parameter>(element::f32, shape);
420 421
        return make_shared<Function>(make_shared<op::Abs>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
422 423
    };

424 425
    auto f = make_graph();
    auto g = make_graph();
426
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
427
    {
428
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
429

430
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
431

432
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
433

434
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
435
    }
436 437
}

438
NGRAPH_TEST(${BACKEND_NAME}, backwards_acos)
439 440 441 442 443 444 445 446 447 448 449 450
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Acos>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
451
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
452 453
}

454
NGRAPH_TEST(${BACKEND_NAME}, backwards_add)
455
{
456
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
457

458
    test::Uniform<float> rng(-1.0f, 1.0f);
459
    Shape shape{2, 3};
460 461
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
462

463
    auto make_graph = [shape]() {
464 465
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
466
        return make_shared<Function>(X0 + X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
467
    };
468
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
469 470
}

471
NGRAPH_TEST(${BACKEND_NAME}, backwards_add_nested)
472
{
473
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
474

475
    test::Uniform<float> rng(-1.0f, 1.0f);
476
    Shape shape{2, 3};
477 478
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
479 480

    auto make_graph = [shape]() {
481 482
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
483 484
        return make_shared<Function>((X0 + X1) + (X1 + X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
485
    };
486
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
487 488
}

489
NGRAPH_TEST(${BACKEND_NAME}, backwards_asin)
490 491 492 493 494 495 496 497 498 499 500 501
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-0.9f, 0.9f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Asin>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
502
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
503 504
}

505
NGRAPH_TEST(${BACKEND_NAME}, backwards_atan)
506 507 508 509 510 511 512 513 514 515 516 517
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng(-10.0f, 10.0f);
    Shape shape{2, 3};
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Atan>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
518
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
519 520
}

521
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast0)
522
{
523
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
524

525
    test::Uniform<float> rng(-1.0f, 1.0f);
526
    Shape shape{3};
527
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
528 529

    auto make_graph = [shape]() {
530
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
531 532 533
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{2, 3}, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
534
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
535 536
}

537
NGRAPH_TEST(${BACKEND_NAME}, backwards_broadcast1)
538
{
539
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
540

541
    test::Uniform<float> rng(-1.0f, 1.0f);
542
    Shape shape{3};
543
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
544 545

    auto make_graph = [shape]() {
546
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
547 548 549
        return make_shared<Function>(make_shared<op::Broadcast>(X0, Shape{3, 2}, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
550
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
551 552
}

553
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_vector)
554
{
555
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
556 557

    test::Uniform<float> rng(-1.0f, 1.0f);
558
    Shape shape_0{3};
559
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
560
    Shape shape_1{2};
561
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
562
    Shape shape_2{1};
563
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
564 565

    auto make_graph = [shape_0, shape_1, shape_2]() {
566 567 568
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
569
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
570 571
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
572 573
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
574 575
}

576
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_0)
577
{
578
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
579 580

    test::Uniform<float> rng(-1.0f, 1.0f);
581
    Shape shape_0{3, 2};
582
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
583
    Shape shape_1{2, 2};
584
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
585
    Shape shape_2{1, 2};
586
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
587 588

    auto make_graph = [shape_0, shape_1, shape_2]() {
589 590 591
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
592
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 0),
593 594
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
595 596
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
597 598
}

599
NGRAPH_TEST(${BACKEND_NAME}, backwards_concat_axis_1)
600
{
601
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
602 603

    test::Uniform<float> rng(-1.0f, 1.0f);
604
    Shape shape_0{2, 3};
605
    auto x0 = rng.initialize(backend->create_tensor(element::f32, shape_0));
606
    Shape shape_1{2, 2};
607
    auto x1 = rng.initialize(backend->create_tensor(element::f32, shape_1));
608
    Shape shape_2{2, 1};
609
    auto x2 = rng.initialize(backend->create_tensor(element::f32, shape_2));
610 611

    auto make_graph = [shape_0, shape_1, shape_2]() {
612 613 614
        auto X0 = make_shared<op::Parameter>(element::f32, shape_0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape_1);
        auto X2 = make_shared<op::Parameter>(element::f32, shape_2);
615
        return make_shared<Function>(make_shared<op::Concat>(NodeVector{X0, X1, X2}, 1),
616 617
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };
618 619
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
620 621
}

622
NGRAPH_TEST(${BACKEND_NAME}, backwards_ceiling)
623
{
624
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
625 626 627 628 629 630

    // The numeric derivative and the symbolic one may disagree near integers, so we will dance around
    // them.
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
631
    Shape shape{2, 3};
632 633

    auto make_graph = [shape]() {
634
        auto X = make_shared<op::Parameter>(element::f32, shape);
635 636
        return make_shared<Function>(make_shared<op::Ceiling>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
637 638
    };

639 640
    auto f = make_graph();
    auto g = make_graph();
641
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
642
    {
643
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
644

645
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
646

647
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
648

649
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
650

651
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
652

653
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
654 655 656
    }
}

657
NGRAPH_TEST(${BACKEND_NAME}, backwards_cos)
658
{
659
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
660 661

    test::Uniform<float> rng(-10.0f, 10.0f);
662
    Shape shape{2, 3};
663
    auto make_graph = [shape]() {
664
        auto X = make_shared<op::Parameter>(element::f32, shape);
665 666
        return make_shared<Function>(make_shared<op::Cos>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
667 668
    };

669 670
    auto f = make_graph();
    auto g = make_graph();
671
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
672
    {
673
        auto x = rng.initialize(backend->create_tensor<float>(shape));
674

675
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
676 677 678
    }
}

679
NGRAPH_TEST(${BACKEND_NAME}, backwards_cosh)
680
{
681
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
682 683

    test::Uniform<float> rng(-10.0f, 10.0f);
684
    Shape shape{2, 3};
685
    auto make_graph = [shape]() {
686
        auto X = make_shared<op::Parameter>(element::f32, shape);
687 688
        return make_shared<Function>(make_shared<op::Cosh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
689 690
    };

691 692
    auto f = make_graph();
    auto g = make_graph();
693
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
694
    {
695
        auto x = rng.initialize(backend->create_tensor<float>(shape));
696

697
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
698 699 700
    }
}

701
NGRAPH_TEST(${BACKEND_NAME}, backwards_divide)
702
{
703
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
704

705 706 707
    test::Uniform<float> rng(-1.0f, 1.0f);
    test::Uniform<float> rng1(1.0f, 2.0f);
    test::Uniform<float> rng2(-2.0f, -1.0f);
708
    Shape shape{2, 3};
709 710 711
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng1.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng2.initialize(backend->create_tensor<float>(shape));
712

713
    auto make_graph = [shape]() {
714 715
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
716
        return make_shared<Function>(X0 / X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
717
    };
718 719
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
720
}
721

722
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_scalar)
723
{
724
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
725

726
    test::Uniform<float> rng(-1.0f, 1.0f);
727 728
    Shape shape0{};
    Shape shape1{};
729 730
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
731 732

    auto make_graph = [shape0, shape1]() {
733 734
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
735 736 737
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
738
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
739 740
}

741
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_scalar_tensor)
742
{
743
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
744

745
    test::Uniform<float> rng(-1.0f, 1.0f);
746 747
    Shape shape0{};
    Shape shape1{3, 4};
748 749
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
750 751

    auto make_graph = [shape0, shape1]() {
752 753
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
754 755 756
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
757
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
758 759
}

760
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_scalar)
761
{
762
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
763

764
    test::Uniform<float> rng(-1.0f, 1.0f);
765 766
    Shape shape0{3, 4};
    Shape shape1{};
767 768
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
769 770

    auto make_graph = [shape0, shape1]() {
771 772
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
773 774 775
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
776
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
777 778
}

779
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_vector_vector)
780
{
781
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
782

783
    test::Uniform<float> rng(-1.0f, 1.0f);
784 785
    Shape shape0{3};
    Shape shape1{3};
786 787
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
788 789

    auto make_graph = [shape0, shape1]() {
790 791
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
792 793 794
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
795
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
796 797
}

798
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor_vector)
799
{
800
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
801

802
    test::Uniform<float> rng(-1.0f, 1.0f);
803 804
    Shape shape0{4, 3};
    Shape shape1{3};
805 806
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
807 808

    auto make_graph = [shape0, shape1]() {
809 810
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
811 812 813
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
814
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
815 816
}

817
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor2_tensor2)
818
{
819
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
820

821
    test::Uniform<float> rng(-1.0f, 1.0f);
822 823
    Shape shape0{4, 3};
    Shape shape1{3, 5};
824 825
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
826 827

    auto make_graph = [shape0, shape1]() {
828 829
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
830 831
        return make_shared<Function>(make_shared<op::Dot>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Adam Procter's avatar
Adam Procter committed
832
    };
833
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Adam Procter's avatar
Adam Procter committed
834 835
}

836
NGRAPH_TEST(${BACKEND_NAME}, backwards_dot_tensor3_tensor3)
Adam Procter's avatar
Adam Procter committed
837
{
838
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Adam Procter's avatar
Adam Procter committed
839 840

    test::Uniform<float> rng(-1.0f, 1.0f);
841 842
    Shape shape0{2, 4, 3};
    Shape shape1{4, 3, 3};
843 844
    auto x0 = rng.initialize(backend->create_tensor<float>(shape0));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape1));
Adam Procter's avatar
Adam Procter committed
845 846

    auto make_graph = [shape0, shape1]() {
847 848
        auto X0 = make_shared<op::Parameter>(element::f32, shape0);
        auto X1 = make_shared<op::Parameter>(element::f32, shape1);
Adam Procter's avatar
Adam Procter committed
849 850
        return make_shared<Function>(make_shared<op::Dot>(X0, X1, 2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
851
    };
852
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
853 854
}

855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
#ifdef AUTODIFF_BACKEND_CPU
NGRAPH_TEST(${BACKEND_NAME}, backwards_batchdot_tensor2_tensor2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
    std::string backend_name = "${BACKEND_NAME}";

    const std::string file_name("mxnet/batch_dot_3.json");
    auto f = make_function_from_file(file_name);

    test::Uniform<float> rng(-1.0f, 1.0f);
    std::vector<std::shared_ptr<ngraph::runtime::Tensor>> args;
    for (shared_ptr<op::Parameter> param : f->get_parameters())
    {
        args.push_back(rng.initialize(backend->create_tensor<float>(param->get_shape())));
    }

    auto g = make_function_from_file(file_name);
    pass::Manager pass_manager;
    pass_manager.register_pass<runtime::cpu::pass::CPUBatchFusion>();
    pass_manager.run_passes(g);
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, args, .01f, .01f));
}
#endif

879
NGRAPH_TEST(${BACKEND_NAME}, backwards_exp)
Scott Cyphers's avatar
Scott Cyphers committed
880
{
881
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
882

883
    test::Uniform<float> rng(-1.0f, 1.0f);
884
    Shape shape{2, 3};
885
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
886 887

    auto make_graph = [shape]() {
888
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
889 890
        return make_shared<Function>(make_shared<op::Exp>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
891
    };
892
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
893 894
}

895
NGRAPH_TEST(${BACKEND_NAME}, backwards_floor)
896
{
897
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
898 899 900 901 902 903

    // The numeric derivative and the symbolic one may disagree near integers, so we will dance around
    // them.
    test::Uniform<float> rng_minusone(-0.95f, -0.05f);
    test::Uniform<float> rng_plusone(0.05f, 0.95f);
    test::Uniform<float> rng_plustwo(1.05f, 1.95f);
904
    Shape shape{2, 3};
905 906

    auto make_graph = [shape]() {
907
        auto X = make_shared<op::Parameter>(element::f32, shape);
908 909
        return make_shared<Function>(make_shared<op::Floor>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
910 911
    };

912 913
    auto f = make_graph();
    auto g = make_graph();
914
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
915
    {
916
        auto x_minusone = rng_minusone.initialize(backend->create_tensor<float>(shape));
917

918
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_minusone}, .01f, .01f));
919

920
        auto x_plusone = rng_plusone.initialize(backend->create_tensor<float>(shape));
921

922
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plusone}, .01f, .01f));
923

924
        auto x_plustwo = rng_plustwo.initialize(backend->create_tensor<float>(shape));
925

926
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_plustwo}, .01f, .01f));
927 928 929
    }
}

930
NGRAPH_TEST(${BACKEND_NAME}, backwards_log)
Scott Cyphers's avatar
Scott Cyphers committed
931
{
932
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
933

934
    test::Uniform<float> rng(1.0f, 2.0f);
935
    Shape shape{2, 3};
936
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
937 938

    auto make_graph = [shape]() {
939
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
940 941
        return make_shared<Function>(make_shared<op::Log>(X0),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
942
    };
943
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
944 945
}

946
NGRAPH_TEST(${BACKEND_NAME}, backwards_maximum)
947
{
948
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
949

950
    test::Uniform<float> rng(-1.0f, 1.0f);
951
    Shape shape{2, 3};
952 953
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
954 955

    auto make_graph = [shape]() {
956 957
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
958 959 960
        return make_shared<Function>(make_shared<op::Maximum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
961
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
962 963
}

964
NGRAPH_TEST(${BACKEND_NAME}, backwards_minimum)
965
{
966
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
967

968
    test::Uniform<float> rng(-1.0f, 1.0f);
969
    Shape shape{2, 3};
970 971
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
972 973

    auto make_graph = [shape]() {
974 975
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
976 977 978
        return make_shared<Function>(make_shared<op::Minimum>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };
979
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
980 981
}

982
NGRAPH_TEST(${BACKEND_NAME}, backwards_multiply)
983
{
984
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
985

986
    test::Uniform<float> rng(-1.0f, 1.0f);
987
    Shape shape{2, 3};
988 989
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
990

991
    auto make_graph = [shape]() {
992 993
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
994
        return make_shared<Function>(X0 * X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
995
    };
996
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
997
}
Scott Cyphers's avatar
Scott Cyphers committed
998

999
NGRAPH_TEST(${BACKEND_NAME}, backwards_negative)
Scott Cyphers's avatar
Scott Cyphers committed
1000
{
1001
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1002

1003
    test::Uniform<float> rng(-1.0f, 1.0f);
1004
    Shape shape{2, 3};
1005
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1006 1007

    auto make_graph = [shape]() {
1008
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1009
        return make_shared<Function>(-X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1010
    };
1011
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1012 1013
}

1014
NGRAPH_TEST(${BACKEND_NAME}, backwards_parameter)
Scott Cyphers's avatar
Scott Cyphers committed
1015
{
1016
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1017

1018
    test::Uniform<float> rng(-1.0f, 1.0f);
1019
    Shape shape{2, 3};
1020
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1021
    auto make_graph = [shape]() {
1022
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1023
        return make_shared<Function>(X0, std::vector<std::shared_ptr<op::Parameter>>{X0});
Scott Cyphers's avatar
Scott Cyphers committed
1024
    };
1025
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
1026 1027
}

1028
NGRAPH_TEST(${BACKEND_NAME}, backwards_power)
1029
{
1030
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1031 1032 1033

    test::Uniform<float> rng_neg(-5.0f, -0.5f);
    test::Uniform<float> rng_pos(0.5f, 5.0f);
1034
    Shape shape{2, 3};
1035 1036

    auto make_graph = [shape]() {
1037 1038
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1039 1040 1041 1042
        return make_shared<Function>(std::make_shared<op::Power>(X0, X1),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
    };

1043 1044
    auto x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_neg.initialize(backend->create_tensor<float>(shape));
1045

1046
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1047

1048 1049
    x0 = rng_pos.initialize(backend->create_tensor<float>(shape));
    x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1050

1051
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
1052 1053
}

1054
NGRAPH_TEST(${BACKEND_NAME}, backwards_relu)
1055
{
1056
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1057 1058 1059 1060

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
1061 1062
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));
1063 1064 1065 1066 1067 1068 1069

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Relu>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1070 1071
    auto f = make_graph();
    auto g = make_graph();
1072 1073
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
1074
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1075

1076
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1077

1078
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1079

1080
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1081 1082 1083
    }
}

1084
NGRAPH_TEST(${BACKEND_NAME}, backwards_replace_slice)
1085
{
1086
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1087 1088

    test::Uniform<float> rng(-10.0f, 10.0f);
1089 1090
    Shape shape_x{5, 5};
    Shape shape_y{2, 2};
1091
    auto make_graph = [shape_x, shape_y]() {
1092 1093
        auto X = make_shared<op::Parameter>(element::f32, shape_x);
        auto Y = make_shared<op::Parameter>(element::f32, shape_y);
1094 1095 1096 1097 1098
        return make_shared<Function>(
            make_shared<op::ReplaceSlice>(X, Y, Coordinate{2, 3}, Coordinate{4, 5}),
            std::vector<std::shared_ptr<op::Parameter>>{X, Y});
    };

1099 1100
    auto f = make_graph();
    auto g = make_graph();
1101
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1102
    {
1103 1104
        auto x = rng.initialize(backend->create_tensor<float>(shape_x));
        auto y = rng.initialize(backend->create_tensor<float>(shape_y));
1105

1106
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x, y}, .01f, .01f));
1107 1108 1109
    }
}

1110
NGRAPH_TEST(${BACKEND_NAME}, backwards_reshape)
1111
{
1112
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1113

1114
    test::Uniform<float> rng(-1.0f, 1.0f);
1115
    Shape shape{3, 4};
1116
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
1117 1118

    auto make_graph = [shape]() {
1119
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
1120 1121 1122
        return make_shared<Function>(make_shared<op::Reshape>(X0, AxisVector{1, 0}, Shape{4, 3}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1123
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1124 1125
}

1126
NGRAPH_TEST(${BACKEND_NAME}, backwards_select)
1127
{
1128
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1129 1130

    test::Uniform<float> rng(-10.0f, 10.0f);
1131
    Shape shape{2, 3};
1132
    auto make_graph = [shape]() {
1133 1134 1135
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1136 1137 1138 1139
        return make_shared<Function>(make_shared<op::Select>(X0, X1, X2),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1140 1141
    auto f = make_graph();
    auto g = make_graph();
1142
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1143
    {
1144
        auto x0 = backend->create_tensor(element::boolean, shape);
1145
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1146 1147 1148 1149
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1150
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1151 1152 1153
    }
}

1154
NGRAPH_TEST(${BACKEND_NAME}, backwards_select_nested)
1155
{
1156
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1157 1158

    test::Uniform<float> rng(-10.0f, 10.0f);
1159
    Shape shape{2, 3};
1160
    auto make_graph = [shape]() {
1161 1162 1163
        auto X0 = make_shared<op::Parameter>(element::boolean, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1164
        return make_shared<Function>(make_shared<op::Select>(X0, X2 + X1, X2 - X1),
1165 1166 1167
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
    };

1168 1169
    auto f = make_graph();
    auto g = make_graph();
1170
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1171
    {
1172
        auto x0 = backend->create_tensor(element::boolean, shape);
1173
        write_vector(x0, vector<char>{0, 1, 0, 1, 0, 1});
1174 1175 1176 1177
        auto x1 = rng.initialize(backend->create_tensor<float>(shape));
        auto x2 = rng.initialize(backend->create_tensor<float>(shape));

        EXPECT_TRUE(autodiff_numeric_compare_selective<float>(
1178
            backend.get(), f, g, {x0, x1, x2}, .01f, .01f, std::vector<bool>{false, true, true}));
1179 1180 1181
    }
}

1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
NGRAPH_TEST(${BACKEND_NAME}, backwards_sigmoid)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
    Shape shape{2, 3};
    auto x0 = rng_neg.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng_pos.initialize(backend->create_tensor<float>(shape));

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Sigmoid>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1198 1199
    auto f = make_graph();
    auto g = make_graph();
1200 1201 1202 1203
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
    {
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));

1204
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1205 1206 1207

        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));

1208
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1209 1210 1211
    }
}

1212
NGRAPH_TEST(${BACKEND_NAME}, backwards_sign)
1213
{
1214
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1215 1216 1217 1218 1219

    // The numeric derivative and the symbolic one may disagree around 0, so we will dance around
    // that point by skipping (-0.01,0.01).
    test::Uniform<float> rng_neg(-1.0f, -0.01f);
    test::Uniform<float> rng_pos(0.01f, 1.0f);
1220
    Shape shape{2, 3};
1221 1222

    auto make_graph = [shape]() {
1223
        auto X = make_shared<op::Parameter>(element::f32, shape);
1224 1225
        return make_shared<Function>(make_shared<op::Sign>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1226 1227
    };

1228 1229
    auto f = make_graph();
    auto g = make_graph();
1230
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1231
    {
1232
        auto x_neg = rng_neg.initialize(backend->create_tensor<float>(shape));
1233

1234
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_neg}, .01f, .01f));
1235

1236
        auto x_pos = rng_pos.initialize(backend->create_tensor<float>(shape));
1237

1238
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_pos}, .01f, .01f));
1239 1240 1241
    }
}

1242
NGRAPH_TEST(${BACKEND_NAME}, backwards_sin)
1243
{
1244
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1245 1246

    test::Uniform<float> rng(-10.0f, 10.0f);
1247
    Shape shape{2, 3};
1248
    auto make_graph = [shape]() {
1249
        auto X = make_shared<op::Parameter>(element::f32, shape);
1250 1251
        return make_shared<Function>(make_shared<op::Sin>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1252 1253
    };

1254 1255
    auto f = make_graph();
    auto g = make_graph();
1256
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1257
    {
1258
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1259

1260
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1261 1262 1263
    }
}

1264
NGRAPH_TEST(${BACKEND_NAME}, backwards_sinh)
1265
{
1266
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1267 1268

    test::Uniform<float> rng(-10.0f, 10.0f);
1269
    Shape shape{2, 3};
1270
    auto make_graph = [shape]() {
1271
        auto X = make_shared<op::Parameter>(element::f32, shape);
1272 1273
        return make_shared<Function>(make_shared<op::Sinh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1274 1275
    };

1276 1277
    auto f = make_graph();
    auto g = make_graph();
1278
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1279
    {
1280
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1281

1282
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1283 1284 1285
    }
}

1286
NGRAPH_TEST(${BACKEND_NAME}, backwards_slice)
1287
{
1288
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1289
    test::Uniform<float> rng(-10.0f, 10.0f);
1290
    Shape shape{5, 5};
1291
    auto make_graph = [shape]() {
1292
        auto X = make_shared<op::Parameter>(element::f32, shape);
1293 1294 1295 1296
        return make_shared<Function>(make_shared<op::Slice>(X, Coordinate{2, 3}, Coordinate{4, 5}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };

1297 1298
    auto f = make_graph();
    auto g = make_graph();
1299
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1300
    {
1301
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1302

1303
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1304 1305 1306
    }
}

1307
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_all)
1308
{
1309
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1310

adstraw's avatar
adstraw committed
1311
    test::Uniform<float> rng(-1.0f, 1.0f);
1312
    Shape shape{2, 3};
1313
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1314

1315
    auto make_graph = [shape]() {
adstraw's avatar
adstraw committed
1316 1317 1318
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
1319
    };
1320
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1321
}
1322

1323
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_axis)
adstraw's avatar
adstraw committed
1324
{
1325
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1326

adstraw's avatar
adstraw committed
1327 1328
    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3};
1329
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1330 1331 1332 1333 1334 1335

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1336
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1337 1338
}

1339
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_underflow)
adstraw's avatar
adstraw committed
1340
{
1341
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1342 1343 1344 1345

    auto low = std::numeric_limits<float>::lowest();

    Shape shape{2, 3};
1346
    auto x0 = backend->create_tensor(element::f32, shape);
adstraw's avatar
adstraw committed
1347 1348 1349 1350 1351 1352 1353
    copy_data(x0, vector<float>{low, 1, 2, 3, 4, 5});

    auto make_graph = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1354
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1355 1356
}

1357
NGRAPH_TEST(${BACKEND_NAME}, backwards_softmax_3d)
adstraw's avatar
adstraw committed
1358
{
1359
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
adstraw's avatar
adstraw committed
1360 1361 1362

    test::Uniform<float> rng(-1.0f, 1.0f);
    Shape shape{2, 3, 4};
1363
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
adstraw's avatar
adstraw committed
1364 1365 1366 1367 1368 1369

    auto make_graph0 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1370
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph0, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1371 1372 1373 1374 1375 1376

    auto make_graph1 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1377
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph1, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1378 1379 1380 1381 1382 1383

    auto make_graph2 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1384
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph2, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1385 1386 1387 1388 1389 1390

    auto make_graph01 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1391
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph01, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1392 1393 1394 1395 1396 1397

    auto make_graph02 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1398
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph02, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1399 1400 1401 1402 1403 1404

    auto make_graph12 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1405
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph12, {x0}, .01f, .01f));
adstraw's avatar
adstraw committed
1406 1407 1408 1409 1410 1411

    auto make_graph012 = [shape]() {
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Softmax>(X0, AxisSet{0, 1, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X0});
    };
1412
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph012, {x0}, .01f, .01f));
1413 1414
}

1415
NGRAPH_TEST(${BACKEND_NAME}, backwards_subtract)
Scott Cyphers's avatar
Scott Cyphers committed
1416
{
1417
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1418

1419
    test::Uniform<float> rng(-1.0f, 1.0f);
1420
    Shape shape{2, 3};
1421 1422
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1423 1424

    auto make_graph = [shape]() {
1425 1426
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
1427
        return make_shared<Function>(X0 - X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
Scott Cyphers's avatar
Scott Cyphers committed
1428
    };
1429
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1430
}
Scott Cyphers's avatar
Scott Cyphers committed
1431

1432
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_v2s)
1433
{
1434
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1435 1436

    test::Uniform<float> rng(-1.0f, 1.0f);
1437
    Shape shape{8};
1438
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1439 1440

    auto make_graph = [shape]() {
1441
        auto X = make_shared<op::Parameter>(element::f32, shape);
1442 1443 1444
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1445
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1446 1447
}

1448
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2s)
1449
{
1450
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1451 1452

    test::Uniform<float> rng(-1.0f, 1.0f);
1453
    Shape shape{8, 9};
1454
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1455 1456

    auto make_graph = [shape]() {
1457
        auto X = make_shared<op::Parameter>(element::f32, shape);
1458 1459 1460
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0, 1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1461
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1462 1463
}

1464
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_0)
1465
{
1466
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1467 1468

    test::Uniform<float> rng(-1.0f, 1.0f);
1469
    Shape shape{8, 9};
1470
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1471 1472

    auto make_graph = [shape]() {
1473
        auto X = make_shared<op::Parameter>(element::f32, shape);
1474 1475 1476
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{0}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1477
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1478 1479
}

1480
NGRAPH_TEST(${BACKEND_NAME}, backwards_sum_m2v_1)
1481
{
1482
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1483 1484

    test::Uniform<float> rng(-1.0f, 1.0f);
1485
    Shape shape{8, 9};
1486
    auto x = rng.initialize(backend->create_tensor<float>(shape));
1487 1488

    auto make_graph = [shape]() {
1489
        auto X = make_shared<op::Parameter>(element::f32, shape);
1490 1491 1492
        return make_shared<Function>(make_shared<op::Sum>(X, AxisSet{1}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1493
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1494 1495
}

1496
NGRAPH_TEST(${BACKEND_NAME}, backwards_tan)
1497
{
1498
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1499 1500 1501 1502

    auto pi = 3.14159f;

    // Stay away from the asymptotes at 6 and 12 o'clock.
1503
    auto slop = 0.2f;
1504 1505 1506
    test::Uniform<float> rng_r(-pi / 2 + slop, pi / 2 - slop);
    test::Uniform<float> rng_l(pi / 2 + slop, (3 * pi) / 2 - slop);

1507
    Shape shape{2, 3};
1508 1509

    auto make_graph = [shape]() {
1510
        auto X = make_shared<op::Parameter>(element::f32, shape);
1511 1512
        return make_shared<Function>(make_shared<op::Tan>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1513 1514
    };

1515 1516
    auto f = make_graph();
    auto g = make_graph();
1517
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1518
    {
1519
        auto x_r = rng_r.initialize(backend->create_tensor<float>(shape));
1520

1521
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_r}, .01f, .01f));
1522

1523
        auto x_l = rng_l.initialize(backend->create_tensor<float>(shape));
1524

1525
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x_l}, .01f, .01f));
1526 1527 1528
    }
}

1529
NGRAPH_TEST(${BACKEND_NAME}, backwards_tanh)
1530
{
1531
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1532 1533

    test::Uniform<float> rng(-10.0f, 10.0f);
1534
    Shape shape{2, 3};
1535
    auto make_graph = [shape]() {
1536
        auto X = make_shared<op::Parameter>(element::f32, shape);
1537 1538
        return make_shared<Function>(make_shared<op::Tanh>(X),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
1539 1540
    };

1541 1542
    auto f = make_graph();
    auto g = make_graph();
1543
    for (auto i = 0; i < ${TEST_LOOPS}; i++)
1544
    {
1545
        auto x = rng.initialize(backend->create_tensor<float>(shape));
1546

1547
        EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), f, g, {x}, .01f, .01f));
1548 1549 1550
    }
}

1551
NGRAPH_TEST(${BACKEND_NAME}, backwards_abc)
Scott Cyphers's avatar
Scott Cyphers committed
1552
{
1553
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
Scott Cyphers's avatar
Scott Cyphers committed
1554

1555
    test::Uniform<float> rng(-1.0f, 1.0f);
1556
    Shape shape{2, 3};
1557 1558 1559
    auto x0 = rng.initialize(backend->create_tensor<float>(shape));
    auto x1 = rng.initialize(backend->create_tensor<float>(shape));
    auto x2 = rng.initialize(backend->create_tensor<float>(shape));
Scott Cyphers's avatar
Scott Cyphers committed
1560 1561

    auto make_graph = [shape]() {
1562 1563 1564
        auto X0 = make_shared<op::Parameter>(element::f32, shape);
        auto X1 = make_shared<op::Parameter>(element::f32, shape);
        auto X2 = make_shared<op::Parameter>(element::f32, shape);
1565 1566
        return make_shared<Function>((X0 + X1) * X2,
                                     std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
Scott Cyphers's avatar
Scott Cyphers committed
1567
    };
1568

1569 1570
    EXPECT_TRUE(
        autodiff_numeric_compare<float>(backend.get(), make_graph, {x0, x1, x2}, .01f, .01f));
Scott Cyphers's avatar
Scott Cyphers committed
1571
}
1572

1573
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_3d_02)
1574
{
1575
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1576 1577

    test::Uniform<float> rng(-1.0f, 1.0f);
1578
    Shape shape{2, 4, 5};
1579
    auto x = rng.initialize(backend->create_tensor(element::f32, shape));
1580 1581 1582 1583 1584 1585

    auto make_graph = [shape]() {
        auto X = make_shared<op::Parameter>(element::f32, shape);
        return make_shared<Function>(make_shared<op::Reverse>(X, AxisSet{0, 2}),
                                     std::vector<std::shared_ptr<op::Parameter>>{X});
    };
1586
    EXPECT_TRUE(autodiff_numeric_compare<float>(backend.get(), make_graph, {x}, .01f, .01f));
1587
}
1588

1589
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n4c1h4w4_kh2kw2_sh1sw1)
1590
{
1591
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1592
    Shape shape_a{4, 1, 4, 4}; // in NCHW
1593 1594 1595 1596 1597
    Shape maxpool_shape{4, 1, 3, 3};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{2, 2};
    auto window_movement_strides = Strides{1, 1};
1598
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1599
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1600
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1601 1602
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1603 1604
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1605 1606

    vector<float> dataInput{11, 65, 44, 28, 31, 33, 21, 66, 40, 49, 69, 57, 47, 30, 24, 27,
pthoreho's avatar
pthoreho committed
1607 1608 1609
                            13, 56, 46, 60, 61, 41, 25, 42, 48, 53, 51, 43, 59, 58, 29, 71,
                            17, 22, 72, 18, 39, 35, 15, 38, 64, 52, 73, 67, 62, 50, 10, 68,
                            45, 63, 16, 14, 55, 54, 37, 20, 36, 12, 70, 34, 19, 26, 32, 23};
1610

1611
    vector<float> expected{// delta
1612 1613 1614
                           0, 8, 0, 0, 0, 0, 0, 4, 0, 8, 16, 0, 0, 0, 0,  0, 0, 4, 0, 4, 8,  0,
                           0, 0, 0, 4, 4, 0, 4, 4, 0, 4, 0,  0, 8, 0, 4,  0, 0, 0, 8, 0, 16, 0,
                           0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 4,  0, 4, 0, 16, 0, 0, 0, 0, 0};
1615 1616 1617 1618 1619 1620

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1621 1622
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
1623 1624 1625
    ASSERT_TRUE(read_vector<float>(output) == expected);
}

1626
NGRAPH_TEST(${BACKEND_NAME}, backwards_maxpool_n2c1h5w5_kh3kw3_sh2sw2)
1627
{
1628
    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1629

1630
    Shape shape_a{1, 2, 5, 5}; // in NCHW
1631 1632 1633 1634 1635
    Shape maxpool_shape{1, 2, 2, 2};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    Shape window_shape{3, 3};
    auto window_movement_strides = Strides{2, 2};
1636
    auto maxpool = make_shared<op::MaxPool>(A, window_shape, window_movement_strides);
1637
    auto f = make_shared<Function>(maxpool, ParameterVector{A});
1638

1639
    shared_ptr<runtime::Tensor> ep = backend->create_tensor(element::f32, maxpool_shape);
1640 1641
    vector<float> dataEp(shape_size(maxpool_shape), 4);

1642 1643
    shared_ptr<runtime::Tensor> input = backend->create_tensor(element::f32, shape_a);
    shared_ptr<runtime::Tensor> output = backend->create_tensor(element::f32, shape_a);
1644 1645

    vector<float> dataInput{58, 15, 51, 35, 18, 47, 31, 32, 52, 21, 36, 38, 57, 54, 25, 45, 23,
pthoreho's avatar
pthoreho committed
1646 1647
                            30, 16, 27, 48, 20, 41, 37, 43, 39, 22, 28, 33, 29, 12, 17, 44, 42,
                            19, 40, 10, 46, 34, 53, 26, 55, 50, 13, 24, 14, 49, 56, 59, 11};
1648

1649
    vector<float> expected{// delta
1650 1651 1652
                           4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0,
                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0,
                           0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0,  4, 4, 0};
1653 1654 1655 1656 1657 1658

    copy_data(ep, dataEp);
    copy_data(input, dataInput);

    auto C = make_shared<op::Parameter>(element::f32, maxpool_shape);
    auto df = autodiff::backprop_function(f);
1659 1660
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {output}, {input, ep});
1661
    ASSERT_TRUE(read_vector<float>(output) == expected);
1662
}
1663

1664
NGRAPH_TEST(${BACKEND_NAME}, backwards_batch_norm_training)
1665
{
1666
    const Shape input_shape{10, 4, 5, 5};
1667 1668 1669 1670
    const Shape channel_shape{input_shape.at(1)};
    const double eps = 1e-3;
    const element::Type& et = element::f32;
    using T = float;
1671

1672 1673
    // Need to keep the output elements for mean and variance from going out of scope
    // and getting freed.
1674 1675
    NodeVector goes;

1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686
    auto make_graph = [&input_shape, &channel_shape, &eps, &et, &goes] {
        auto input = make_shared<op::Parameter>(et, input_shape);
        auto gamma = make_shared<op::Parameter>(et, channel_shape);
        auto beta = make_shared<op::Parameter>(et, channel_shape);
        auto BN = make_shared<op::BatchNormTraining>(input, gamma, beta, eps);
        auto normed_input = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 0));
        auto mean = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 1));
        auto variance = make_shared<op::Result>(make_shared<op::GetOutputElement>(BN, 2));
        goes.push_back(mean);
        goes.push_back(variance);
        // TODO autodiff testing with more than one result
1687 1688
        auto f =
            make_shared<Function>(ResultVector{normed_input}, ParameterVector{input, gamma, beta});
1689 1690 1691 1692
        return f;
    };

    auto backend = runtime::Backend::create("${BACKEND_NAME}");
1693
    test::Uniform<T> rng(-5.0, 2.0);
1694 1695 1696
    auto input = rng.initialize(backend->create_tensor<T>(input_shape));
    auto gamma = rng.initialize(backend->create_tensor<T>(channel_shape));
    auto beta = rng.initialize(backend->create_tensor<T>(channel_shape));
1697

1698
    EXPECT_TRUE(
1699
        autodiff_numeric_compare<T>(backend.get(), make_graph, {input, gamma, beta}, .005, .005));
1700 1701
}

1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n3_c2_h3)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{3, 2, 3};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{2};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 1;
    size_t sequence_axis = 0;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1715
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1716

1717 1718 1719 1720 1721
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1722

1723
    // input values don't matter
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
    vector<int> va(shape_size(shape), 0);

    vector<int> vc{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
    vector<int> expected{13, 14, 15, 16, 17, 18, 7, 8, 9, 10, 11, 12, 1, 2, 3, 4, 5, 6};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{3, 3};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1737 1738
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {da, db}, {a, b, c});
1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
    ASSERT_EQ(read_vector<int>(da), expected);
}

NGRAPH_TEST(${BACKEND_NAME}, backwards_reverse_sequence_n4d2c3h2w2)
{
    auto backend = runtime::Backend::create("${BACKEND_NAME}");

    Shape shape{4, 2, 3, 2, 2};
    auto A = make_shared<op::Parameter>(element::i32, shape);
    Shape seq_len_shape{4};
    auto B = make_shared<op::Parameter>(element::i32, seq_len_shape);

    size_t batch_axis = 0;
    size_t sequence_axis = 2;

    auto rs = std::make_shared<op::ReverseSequence>(A, B, batch_axis, sequence_axis);
1755
    auto f = make_shared<Function>(rs, ParameterVector{A, B});
1756

1757 1758 1759 1760 1761
    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::i32, seq_len_shape);
    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> da = backend->create_tensor(element::i32, shape);
    shared_ptr<runtime::Tensor> db = backend->create_tensor(element::i32, seq_len_shape);
1762

1763
    // input values don't matter
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
    vector<int> va(shape_size(shape), 0);

    std::vector<int> vc{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                        16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
                        32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
                        48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                        64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
                        80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95};

    std::vector<int> expected{0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
                              16, 17, 18, 19, 20, 21, 22, 23, 28, 29, 30, 31, 24, 25, 26, 27,
                              32, 33, 34, 35, 40, 41, 42, 43, 36, 37, 38, 39, 44, 45, 46, 47,
                              48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
                              64, 65, 66, 67, 68, 69, 70, 71, 76, 77, 78, 79, 72, 73, 74, 75,
                              80, 81, 82, 83, 88, 89, 90, 91, 84, 85, 86, 87, 92, 93, 94, 95};

    copy_data(c, vc);
    copy_data(a, va);

    std::vector<int> seq_lenghts{1, 2, 1, 2};
    copy_data(b, seq_lenghts);

    auto C = make_shared<op::Parameter>(element::i32, shape);
    auto df = autodiff::backprop_function(f);
1788 1789
    auto handle = backend->compile(df);
    backend->call_with_validate(handle, {da, db}, {a, b, c});
1790 1791
    ASSERT_EQ(read_vector<int>(da), expected);
}
1792 1793 1794 1795 1796 1797

// clang-format off
#ifdef AUTODIFF_BACKEND_${BACKEND_NAME}
#undef AUTODIFF_BACKEND_${BACKEND_NAME}
#endif
// clang-format on