reshape_sinking.cpp 12.6 KB
Newer Older
1
//*****************************************************************************
2
// Copyright 2017-2019 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************

#include <algorithm>
#include <cstdio>
#include <iostream>
#include <list>
#include <memory>

#include "gtest/gtest.h"
#include "ngraph/autodiff/adjoints.hpp"
#include "ngraph/file_util.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/log.hpp"
#include "ngraph/ngraph.hpp"
#include "ngraph/op/batch_norm.hpp"
#include "ngraph/op/get_output_element.hpp"
#include "ngraph/op/parameter.hpp"
#include "ngraph/pass/core_fusion.hpp"
#include "ngraph/pass/cse.hpp"
#include "ngraph/pass/manager.hpp"
#include "ngraph/pass/reshape_elimination.hpp"
36
#include "ngraph/pass/reshape_sinking.hpp"
37 38 39 40 41 42 43 44 45 46 47 48 49 50
#include "ngraph/pass/visualize_tree.hpp"
#include "ngraph/serializer.hpp"
#include "ngraph/util.hpp"
#include "nlohmann/json.hpp"
#include "util/all_close.hpp"
#include "util/autodiff/backprop_function.hpp"
#include "util/autodiff/numeric_compare.hpp"
#include "util/ndarray.hpp"
#include "util/random.hpp"
#include "util/test_tools.hpp"

using namespace ngraph;
using namespace std;

51
TEST(reshape_sinking, edge_splitting)
52 53 54 55 56 57 58 59 60
{
    //checks if Reshapes are pushed through op::Abs, but stopped by Sum
    Shape shape_nhwc{16, 28, 28, 1};
    Shape shape_nchw{16, 1, 28, 28};
    auto a = make_shared<op::Parameter>(element::i32, shape_nhwc);
    auto reshape = make_shared<op::Reshape>(a, AxisVector{0, 3, 1, 2}, shape_nchw);
    auto absn = make_shared<op::Abs>(reshape);
    auto absn2 = make_shared<op::Abs>(absn);
    auto sum = make_shared<op::Sum>(reshape, AxisSet{0, 1, 2, 3});
61
    auto func = make_shared<Function>(NodeVector{absn2, sum}, ParameterVector{a});
62 63
    pass::Manager pass_manager;
    //size_t before_count = count_ops_of_type<op::Reshape>(func);
64
    pass_manager.register_pass<pass::VisualizeTree>("before.png");
65
    pass_manager.register_pass<pass::ReshapeSinking>();
66 67
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
68
    pass_manager.register_pass<pass::VisualizeTree>("after.png");
69 70 71 72 73 74 75 76
    pass_manager.run_passes(func);
    ASSERT_EQ(func->get_results().at(1)->get_argument(0), sum);
    auto new_reshape =
        std::dynamic_pointer_cast<op::Reshape>(func->get_results().at(0)->get_argument(0));
    ASSERT_TRUE(new_reshape);
    ASSERT_EQ(new_reshape->get_shape(), shape_nchw);
}

77
TEST(reshape_sinking, broadcast_swimming)
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
{
    Shape shape_nchw{1, 32, 536, 536};
    Shape shape_nhwc{1, 536, 536, 32};
    Shape shape_weights{16, 32, 3, 3};
    Shape conv_nhwc{1, 534, 534, 16};
    Shape conv_nchw{1, 16, 534, 534};
    AxisVector to_nhwc{0, 2, 3, 1};
    AxisVector to_nchw{0, 3, 1, 2};

    size_t channel = 16;
    auto bias = make_shared<op::Parameter>(element::i32, Shape{channel});
    auto bias_reshape = make_shared<op::Reshape>(bias, AxisVector{0}, Shape{1, channel});
    auto bias_broadcast = make_shared<op::Broadcast>(bias_reshape, conv_nhwc, AxisSet{1, 2});

    auto input = make_shared<op::Parameter>(element::i32, shape_nhwc);
    auto reshape_input = make_shared<op::Reshape>(input, to_nchw, shape_nchw);

    auto weights = make_shared<op::Parameter>(element::i32, shape_weights);
    auto conv = make_shared<op::Convolution>(reshape_input, weights);
    auto conv_reshape = make_shared<op::Reshape>(conv, to_nhwc, conv_nhwc);
    auto add = bias_broadcast + conv_reshape;
    auto relu = make_shared<op::Relu>(add);

    auto func = make_shared<Function>(NodeVector{relu}, ParameterVector{bias, input, weights});
    pass::Manager pass_manager;

104
    pass_manager.register_pass<pass::ReshapeSinking>();
105 106 107 108 109 110 111 112 113
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
    pass_manager.run_passes(func);

    ASSERT_EQ(add->get_shape(), conv_nchw);
    ASSERT_EQ(add->get_argument(0)->get_shape(), conv_nchw);
    ASSERT_EQ(add->get_argument(1), conv);
}

114
TEST(reshape_sinking, mnist_conv)
115 116 117 118 119 120 121
{
    const string json_path = file_util::path_join(SERIALIZED_ZOO, "tf_conv_mnist_nhwc.json");
    const string json_string = file_util::read_file_to_string(json_path);
    stringstream ss(json_string);
    shared_ptr<Function> func = ngraph::deserialize(ss);
    pass::Manager pass_manager;
    size_t before_count = count_ops_of_type<op::Reshape>(func);
122
    //pass_manager.register_pass<pass::VisualizeTree>("before.png");
123
    pass_manager.register_pass<pass::ReshapeSinking>();
124 125 126 127
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
    //pass_manager.register_pass<pass::CoreFusion>();
    //pass_manager.register_pass<runtime::cpu::pass::CPUFusion>();
128
    //pass_manager.register_pass<pass::VisualizeTree>("after.png");
129 130 131 132
    pass_manager.run_passes(func);
    size_t before_after = count_ops_of_type<op::Reshape>(func);
    ASSERT_LE(before_after, before_count);
}
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156

TEST(reshape_sinking, nasnet_pooladd)
{
    Shape input_shape{1, 3, 3, 1};

    auto input_type = element::f32;
    auto output_type = element::f32;

    auto X = make_shared<op::Parameter>(input_type, input_shape);
    auto c_weights = op::Constant::create(input_type, Shape{1, 1, 1, 1}, {3});
    auto reshape1 = make_shared<op::Reshape>(X, AxisVector{0, 3, 1, 2}, Shape{1, 1, 3, 3});
    auto avgpool =
        make_shared<op::AvgPool>(reshape1, Shape{1, 1}, Strides{1, 1}, Shape{0, 0}, Shape{0, 0});
    auto reshape2 = make_shared<op::Reshape>(avgpool, AxisVector{0, 2, 3, 1}, Shape{1, 3, 3, 1});
    auto maxpool =
        make_shared<op::MaxPool>(reshape1, Shape{1, 1}, Strides{1, 1}, Shape{0, 0}, Shape{0, 0});
    auto reshape3 = make_shared<op::Reshape>(maxpool, AxisVector{0, 2, 3, 1}, Shape{1, 3, 3, 1});
    auto const1 = op::Constant::create(input_type, Shape{1, 3, 3, 1}, {3});
    auto add1 = make_shared<op::Add>(reshape3, const1);
    auto add2 = make_shared<op::Add>(add1, reshape2);
    auto func = make_shared<Function>(add2, ParameterVector{X});

    pass::Manager pass_manager;
    size_t before_count = count_ops_of_type<op::Reshape>(func);
157
    pass_manager.register_pass<pass::VisualizeTree>("before.png");
158 159 160
    pass_manager.register_pass<pass::ReshapeSinking>();
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
161
    pass_manager.register_pass<pass::VisualizeTree>("after.png");
162 163 164 165
    pass_manager.run_passes(func);
    size_t before_after = count_ops_of_type<op::Reshape>(func);
    ASSERT_LE(before_after, before_count);
}
nikolay.korovaiko's avatar
nikolay.korovaiko committed
166 167 168

TEST(reshape_sinking, slice_pad)
{
169
    Shape shape_a{100, 8, 8, 1};
nikolay.korovaiko's avatar
nikolay.korovaiko committed
170 171 172 173 174 175 176

    AxisVector to_nhwc{0, 2, 3, 1};
    AxisVector to_nchw{0, 3, 1, 2};

    auto A = make_shared<op::Parameter>(element::f32, shape_a);
    auto pad_value = op::Constant::create<float>(element::f32, Shape{}, std::vector<float>{0.0f});

177 178
    CoordinateDiff padding_below{0, 0, 0, 0};
    CoordinateDiff padding_above{0, 1, 1, 0};
nikolay.korovaiko's avatar
nikolay.korovaiko committed
179

180
    auto reshape1 = make_shared<op::Reshape>(A, to_nchw, Shape{100, 1, 8, 8});
nikolay.korovaiko's avatar
nikolay.korovaiko committed
181 182
    auto maxpool =
        make_shared<op::MaxPool>(reshape1, Shape{1, 1}, Strides{2, 2}, Shape{0, 0}, Shape{0, 0});
183
    auto reshape2 = make_shared<op::Reshape>(maxpool, to_nhwc, Shape{100, 4, 4, 1});
184
    auto pad = make_shared<op::Pad>(reshape2, pad_value, padding_below, padding_above);
nikolay.korovaiko's avatar
nikolay.korovaiko committed
185
    auto slice = make_shared<op::Slice>(
186
        pad, Coordinate{0, 1, 1, 0}, Coordinate{100, 5, 5, 1}, Strides{1, 1, 1, 1});
nikolay.korovaiko's avatar
nikolay.korovaiko committed
187

188
    auto reshape3 = make_shared<op::Reshape>(slice, to_nchw, Shape{100, 1, 4, 4});
nikolay.korovaiko's avatar
nikolay.korovaiko committed
189
    auto avgpool = make_shared<op::AvgPool>(reshape3, Shape{1, 1}, Strides{2, 2});
190
    auto reshape4 = make_shared<op::Reshape>(avgpool, to_nhwc, Shape{100, 1, 2, 2});
nikolay.korovaiko's avatar
nikolay.korovaiko committed
191 192 193 194
    auto f = make_shared<Function>(reshape4, ParameterVector{A});

    pass::Manager pass_manager;
    size_t before_count = count_ops_of_type<op::Reshape>(f);
195
    pass_manager.register_pass<pass::VisualizeTree>("before.png");
nikolay.korovaiko's avatar
nikolay.korovaiko committed
196 197 198
    pass_manager.register_pass<pass::ReshapeSinking>();
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
199
    pass_manager.register_pass<pass::VisualizeTree>("after.png");
nikolay.korovaiko's avatar
nikolay.korovaiko committed
200 201 202
    pass_manager.run_passes(f);
    size_t before_after = count_ops_of_type<op::Reshape>(f);
    ASSERT_LE(before_after, before_count);
203
}
nikolay.korovaiko's avatar
nikolay.korovaiko committed
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267

TEST(reshape_sinking, concat)
{
    Shape shape{};
    Shape shape_w{1, 1, 1, 1};
    Shape shape_x{1, 3, 3, 1};
    Shape shape_b{1, 3, 3, 1};
    Shape r_shape{1, 3, 3, 2};

    auto B_ = op::Constant::create(element::f32, shape_w, {3});
    auto B = make_shared<op::Reshape>(B_, AxisVector{3, 2, 0, 1}, Shape{1, 1, 1, 1}); /* nchw */
    auto A_ = make_shared<op::Parameter>(element::f32, shape_x);
    auto A = make_shared<op::Reshape>(A_, AxisVector{0, 3, 1, 2}, Shape{1, 1, 3, 3}); /* nchw */
    auto C = op::Constant::create(element::f32, Shape{1}, {2});
    auto R = make_shared<op::Parameter>(element::f32, r_shape);

    auto conv = make_shared<op::Convolution>(A,
                                             B,
                                             Strides{1, 1},
                                             Strides{1, 1},
                                             CoordinateDiff{0, 0},
                                             CoordinateDiff{0, 0},
                                             Strides{1, 1});
    auto reshape_conv =
        make_shared<op::Reshape>(conv, AxisVector{0, 2, 3, 1}, Shape{1, 3, 3, 1}); /* nhwc */
    auto broadcast = make_shared<op::Broadcast>(C, reshape_conv->get_shape(), AxisSet{0, 1, 2});
    auto add = broadcast + reshape_conv;

    auto B1_ = op::Constant::create(element::f32, shape_w, {3});
    auto B1 = make_shared<op::Reshape>(B1_, AxisVector{3, 2, 0, 1}, Shape{1, 1, 1, 1});
    auto A1_ = make_shared<op::Parameter>(element::f32, shape_x);
    auto A1 = make_shared<op::Reshape>(A1_, AxisVector{0, 3, 1, 2}, Shape{1, 1, 3, 3});
    auto C1 = op::Constant::create(element::f32, Shape{1}, {2});
    auto R1 = make_shared<op::Parameter>(element::f32, r_shape);

    auto conv1 = make_shared<op::Convolution>(A1,
                                              B1,
                                              Strides{1, 1},
                                              Strides{1, 1},
                                              CoordinateDiff{0, 0},
                                              CoordinateDiff{0, 0},
                                              Strides{1, 1});
    auto reshape_conv1 = make_shared<op::Reshape>(conv1, AxisVector{0, 2, 3, 1}, Shape{1, 3, 3, 1});
    auto broadcast1 = make_shared<op::Broadcast>(C1, reshape_conv->get_shape(), AxisSet{0, 1, 2});
    auto add1 = broadcast1 + reshape_conv1;

    auto concat = make_shared<op::Concat>(NodeVector{add, add1}, 3);
    auto relu = make_shared<op::Relu>(concat);
    auto reshape_relu =
        make_shared<op::Reshape>(relu, AxisVector{0, 3, 1, 2}, Shape{1, 2, 3, 3}); /* nchw */
    auto B2_ = op::Constant::create(element::f32, Shape{1, 1, 2, 1}, {2});
    auto B2 = make_shared<op::Reshape>(B2_, AxisVector{3, 2, 0, 1}, Shape{1, 2, 1, 1});
    auto conv2 = make_shared<op::Convolution>(reshape_relu,
                                              B2,
                                              Strides{1, 1},
                                              Strides{1, 1},
                                              CoordinateDiff{0, 0},
                                              CoordinateDiff{0, 0},
                                              Strides{1, 1});
    auto reshape_conv2 =
        make_shared<op::Reshape>(conv2, AxisVector{0, 2, 3, 1}, Shape{1, 3, 3, 1}); /* nhwc */
    auto f = make_shared<Function>(reshape_conv2, ParameterVector{A_, A1_});
    pass::Manager pass_manager;
    size_t before_count = count_ops_of_type<op::Reshape>(f);
268
    pass_manager.register_pass<pass::VisualizeTree>("before.png");
nikolay.korovaiko's avatar
nikolay.korovaiko committed
269 270 271
    pass_manager.register_pass<pass::ReshapeSinking>();
    pass_manager.register_pass<pass::ReshapeElimination>();
    pass_manager.register_pass<pass::CommonSubexpressionElimination>();
272
    pass_manager.register_pass<pass::VisualizeTree>("after.png");
nikolay.korovaiko's avatar
nikolay.korovaiko committed
273 274 275 276
    pass_manager.run_passes(f);
    size_t before_after = count_ops_of_type<op::Reshape>(f);
    ASSERT_LE(before_after, before_count);
}