Commit c775f01b authored by Robert Kimball's avatar Robert Kimball

add unit test to check if output is set correctly

parent 431f245c
......@@ -21,6 +21,7 @@ set (SRC
descriptor/tensor.cpp
descriptor/tensor_view.cpp
descriptor/tuple.cpp
file_util.cpp
function.cpp
log.cpp
node.cpp
......@@ -53,7 +54,6 @@ set (SRC
ops/tuple.cpp
ops/unary_elementwise_arithmetic.cpp
ops/unary_elementwise.cpp
pass/collect_functions.cpp
pass/dump_sorted.cpp
pass/liveness.cpp
pass/manager.cpp
......
......@@ -61,7 +61,7 @@ public:
size_t get_pool_offset() const;
const element::Type& get_element_type() const { return m_element_type; }
static std::string make_tensor_name(const Node* node, size_t value_index);
void set_is_output() { m_is_output = true; }
protected:
const element::Type& m_element_type;
PrimaryTensorView* m_primary_tensor_view;
......
/*
Copyright 2016 Nervana Systems Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <cassert>
#include <dirent.h>
#include <fcntl.h>
#include <fstream>
#include <ftw.h>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <vector>
#include "file_util.hpp"
using namespace std;
string ngraph::file_util::path_join(const string& s1, const string& s2)
{
string rc;
if (s2.size() > 0)
{
if (s2[0] == '/')
{
rc = s2;
}
else if (s1.size() > 0)
{
rc = s1;
if (rc[rc.size() - 1] != '/')
{
rc += "/";
}
rc += s2;
}
else
{
rc = s2;
}
}
else
{
rc = s1;
}
return rc;
}
size_t ngraph::file_util::get_file_size(const string& filename)
{
// ensure that filename exists and get its size
struct stat stats;
if (stat(filename.c_str(), &stats) == -1)
{
throw std::runtime_error("Could not find file: \"" + filename + "\"");
}
return stats.st_size;
}
void ngraph::file_util::remove_directory(const string& dir)
{
file_util::iterate_files(dir,
[](const string& file, bool is_dir) {
if (is_dir)
rmdir(file.c_str());
else
remove(file.c_str());
},
true);
rmdir(dir.c_str());
}
void ngraph::file_util::remove_file(const string& file)
{
remove(file.c_str());
}
bool ngraph::file_util::make_directory(const string& dir)
{
if (mkdir(dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH))
{
if (errno == EEXIST)
{
// not really an error, the directory already exists
return false;
}
throw std::runtime_error("error making directory " + dir + " " + strerror(errno));
}
return true;
}
string ngraph::file_util::make_temp_directory(const string& path)
{
string fname = path.empty() ? file_util::get_temp_directory() : path;
string tmp_template = file_util::path_join(fname, "aeonXXXXXX");
char* tmpname = strdup(tmp_template.c_str());
mkdtemp(tmpname);
string rc = tmpname;
free(tmpname);
return rc;
}
std::string ngraph::file_util::get_temp_directory()
{
const vector<string> potential_tmps = {"NERVANA_AEON_TMP", "TMPDIR", "TMP", "TEMP", "TEMPDIR"};
const char* path = nullptr;
for (const string& var : potential_tmps)
{
path = getenv(var.c_str());
if (path != nullptr)
{
break;
}
}
if (path == nullptr)
{
path = "/tmp";
}
return path;
}
vector<char> ngraph::file_util::read_file_contents(const string& path)
{
size_t file_size = get_file_size(path);
vector<char> data;
data.reserve(file_size);
data.resize(file_size);
FILE* f = fopen(path.c_str(), "rb");
if (f)
{
char* p = data.data();
size_t remainder = file_size;
size_t offset = 0;
while (remainder > 0)
{
size_t rc = fread(&p[offset], 1, remainder, f);
offset += rc;
remainder -= rc;
}
fclose(f);
}
else
{
throw std::runtime_error("error opening file '" + path + "'");
}
return data;
}
std::string ngraph::file_util::read_file_to_string(const std::string& path)
{
std::ifstream f(path);
std::stringstream ss;
ss << f.rdbuf();
return ss.str();
}
void ngraph::file_util::iterate_files(const string& path,
std::function<void(const string& file, bool is_dir)> func,
bool recurse)
{
vector<string> files;
vector<string> dirs;
file_util::iterate_files_worker(path,
[&files, &dirs](const string& file, bool is_dir) {
if (is_dir)
dirs.push_back(file);
else
files.push_back(file);
},
true);
for (auto f : files)
func(f, false);
for (auto f : dirs)
func(f, true);
}
void ngraph::file_util::iterate_files_worker(
const string& path, std::function<void(const string& file, bool is_dir)> func, bool recurse)
{
DIR* dir;
struct dirent* ent;
if ((dir = opendir(path.c_str())) != nullptr)
{
while ((ent = readdir(dir)) != nullptr)
{
string name = ent->d_name;
switch (ent->d_type)
{
case DT_DIR:
if (recurse && name != "." && name != "..")
{
string dir_path = file_util::path_join(path, name);
iterate_files(dir_path, func, recurse);
func(dir_path, true);
}
break;
case DT_LNK: break;
case DT_REG:
name = file_util::path_join(path, name);
func(name, false);
break;
default: break;
}
}
closedir(dir);
}
else
{
throw std::runtime_error("error enumerating file " + path);
}
}
string ngraph::file_util::tmp_filename(const string& extension)
{
string tmp_template =
file_util::path_join(file_util::get_temp_directory(), "ngraph_XXXXXX" + extension);
char* tmpname = strdup(tmp_template.c_str());
// mkstemp opens the file with open() so we need to close it
close(mkstemps(tmpname, static_cast<int>(extension.size())));
string rc = tmpname;
free(tmpname);
return rc;
}
void ngraph::file_util::touch(const std::string& filename)
{
// inspired by http://chris-sharpe.blogspot.com/2013/05/better-than-systemtouch.html
int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK, 0666);
assert(fd >= 0);
close(fd);
// update timestamp for filename
int rc = utimes(filename.c_str(), nullptr);
assert(!rc);
}
bool ngraph::file_util::exists(const std::string& filename)
{
struct stat buffer;
return (stat(filename.c_str(), &buffer) == 0);
}
int ngraph::file_util::try_get_lock(const std::string& filename)
{
mode_t m = umask(0);
int fd = open(filename.c_str(), O_RDWR | O_CREAT, 0666);
umask(m);
if (fd >= 0 && flock(fd, LOCK_EX | LOCK_NB) < 0)
{
close(fd);
fd = -1;
}
return fd;
}
void ngraph::file_util::release_lock(int fd, const std::string& filename)
{
if (fd >= 0)
{
remove_file(filename);
close(fd);
}
}
/*
Copyright 2016 Nervana Systems Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#pragma once
#include <functional>
#include <string>
#include <vector>
namespace ngraph
{
class file_util;
}
class ngraph::file_util
{
public:
static std::string path_join(const std::string& s1, const std::string& s2);
static size_t get_file_size(const std::string& filename);
static void remove_directory(const std::string& dir);
static bool make_directory(const std::string& dir);
static std::string make_temp_directory(const std::string& path = "");
static std::string get_temp_directory();
static void remove_file(const std::string& file);
static std::vector<char> read_file_contents(const std::string& path);
static std::string read_file_to_string(const std::string& path);
static void iterate_files(const std::string& path,
std::function<void(const std::string& file, bool is_dir)> func,
bool recurse = false);
static std::string tmp_filename(const std::string& extension = "");
static void touch(const std::string& filename);
static bool exists(const std::string& filename);
static int try_get_lock(const std::string& filename);
static void release_lock(int fd, const std::string& filename);
private:
static void iterate_files_worker(const std::string& path,
std::function<void(const std::string& file, bool is_dir)> func,
bool recurse = false);
};
......@@ -33,6 +33,7 @@ Function::Function(const std::shared_ptr<Node>& result,
, m_ordered_ops_valid(false)
, m_instance_id(m_next_instance_id.fetch_add(1))
{
m_result->set_is_output();
traverse_nodes(this, [&](shared_ptr<Node> node) { m_ops.push_back(node); });
}
......
......@@ -118,6 +118,10 @@ bool Node::is_output() const
void Node::set_is_output()
{
m_is_output = true;
for (descriptor::Output& output : get_outputs())
{
output.get_tensor().set_is_output();
}
}
std::string Node::get_node_id() const
......
// ----------------------------------------------------------------------------
// Copyright 2017 Nervana Systems Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// ----------------------------------------------------------------------------
#include "ngraph/pass/collect_functions.hpp"
#include "ngraph/function.hpp"
#include "ngraph/log.hpp"
#include "ngraph/node.hpp"
#include "ngraph/ops/function_call.hpp"
#include "ngraph/ops/op.hpp"
#include "ngraph/util.hpp"
using namespace std;
using namespace ngraph;
using namespace ngraph::pass;
bool CollectFunctions::run_on_function(shared_ptr<ngraph::Function> func)
{
set<shared_ptr<ngraph::Function>> functions;
deque<shared_ptr<ngraph::Function>> stack;
stack.push_back(func);
while (stack.empty() == false)
{
shared_ptr<ngraph::Function> f = stack.front();
stack.pop_front();
functions.insert(f);
traverse_nodes(f, [&](shared_ptr<Node> node) {
shared_ptr<op::FunctionCall> fc = dynamic_pointer_cast<op::FunctionCall>(node);
if (fc)
{
stack.push_back(fc->get_function());
}
});
}
get_state().set_functions(functions);
return false;
}
// ----------------------------------------------------------------------------
// Copyright 2017 Nervana Systems Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// ----------------------------------------------------------------------------
#pragma once
#include "ngraph/pass/pass.hpp"
namespace ngraph
{
namespace pass
{
class CollectFunctions;
}
}
class ngraph::pass::CollectFunctions : public FunctionPass
{
public:
bool run_on_function(std::shared_ptr<ngraph::Function>) override;
private:
};
......@@ -35,6 +35,9 @@ bool pass::DumpSorted::run_on_module(vector<shared_ptr<ngraph::Function>>& funct
{
for (shared_ptr<Function> f : functions)
{
out << "=====================================================================\n";
out << f->get_name() << " start\n";
out << "=====================================================================\n";
for (const shared_ptr<Node>& node : f->get_ordered_ops())
{
out << node->get_name() << "(";
......@@ -67,6 +70,9 @@ bool pass::DumpSorted::run_on_module(vector<shared_ptr<ngraph::Function>>& funct
out << " F " << tensor->get_name() << "\n";
}
}
out << "=====================================================================\n";
out << f->get_name() << " end\n";
out << "=====================================================================\n";
}
}
......
......@@ -18,6 +18,8 @@
#include "ngraph/function.hpp"
#include "ngraph/log.hpp"
#include "ngraph/node.hpp"
#include "ngraph/ops/function_call.hpp"
#include "ngraph/ops/reduce.hpp"
#include "ngraph/pass/manager.hpp"
#include "ngraph/pass/pass.hpp"
......@@ -36,10 +38,37 @@ void ngraph::pass::Manager::initialize_default_passes()
{
}
static void find_functions(shared_ptr<Function> f, set<shared_ptr<Function>>& funcs)
{
funcs.insert(f);
for (shared_ptr<Node> node : f->get_ops())
{
shared_ptr<op::FunctionCall> fc = dynamic_pointer_cast<op::FunctionCall>(node);
if (fc)
{
find_functions(fc->get_function(), funcs);
}
shared_ptr<op::Reduce> reduce = dynamic_pointer_cast<op::Reduce>(node);
if (reduce)
{
find_functions(reduce->get_reduction_function(), funcs);
}
}
}
void ngraph::pass::Manager::run_passes(shared_ptr<Function> func)
{
vector<shared_ptr<Function>> fs = {func};
get_state().set_functions(fs);
// find all functions
set<shared_ptr<Function>> tfs;
find_functions(func, tfs);
get_state().set_functions(tfs);
vector<shared_ptr<Function>> fs;
for (shared_ptr<Function> f : get_state().get_functions())
{
fs.push_back(f);
}
for (shared_ptr<PassBase> pass : m_pass_list)
{
......@@ -77,40 +106,6 @@ void ngraph::pass::Manager::run_passes(shared_ptr<Function> func)
}
}
}
// for (shared_ptr<ModulePass>& p : m_module_passes)
// {
// p->set_state(get_state());
// p->run_on_module(fs);
// }
// for (Function* f : fs)
// {
// for (shared_ptr<FunctionPass> p : m_function_passes)
// {
// p->set_state(get_state());
// p->run_on_function(f);
// }
// }
// for (Function* f : fs)
// {
// NGRAPH_INFO;
// for (shared_ptr<NodePass> p : m_node_passes)
// {
// for (Node* node : f->get_ops())
// {
// NGRAPH_INFO;
// p->set_state(get_state());
// p->run_on_node(node);
// }
// }
// }
// for (shared_ptr<CallGraphPass>& p : m_call_graph_passes)
// {
// p->set_state(get_state());
// p->run_on_call_graph(func->get_ordered_ops());
// }
}
ngraph::pass::ManagerState& ngraph::pass::Manager::get_state()
......
......@@ -23,7 +23,7 @@
using namespace std;
using namespace ngraph;
vector<shared_ptr<Function>>& ngraph::pass::ManagerState::get_functions()
const vector<shared_ptr<Function>>& ngraph::pass::ManagerState::get_functions()
{
return m_function_list;
}
......
......@@ -30,7 +30,7 @@ namespace ngraph
class ngraph::pass::ManagerState
{
public:
std::vector<std::shared_ptr<Function>>& get_functions();
const std::vector<std::shared_ptr<Function>>& get_functions();
template <typename T>
void set_functions(const T& collection)
......
......@@ -20,10 +20,8 @@
using namespace std;
using namespace ngraph::runtime::cpu;
CallFrame::CallFrame(EntryPoint compiled_function,
const std::vector<std::shared_ptr<CallFrame>>& callees)
CallFrame::CallFrame(EntryPoint compiled_function)
: m_compiled_function(compiled_function)
, m_callees(callees)
{
}
......@@ -31,23 +29,23 @@ void CallFrame::tensor_call(
const std::vector<std::shared_ptr<ngraph::runtime::TensorView>>& input_tvs,
const std::vector<std::shared_ptr<ngraph::runtime::TensorView>>& output_tvs)
{
m_inputs.clear();
m_outputs.clear();
vector<void*> inputs;
vector<void*> outputs;
for (size_t i = 0; i < input_tvs.size(); i++)
{
shared_ptr<runtime::cpu::CPUTensorView> tv =
static_pointer_cast<runtime::cpu::CPUTensorView>(input_tvs[i]);
m_inputs.push_back(tv->get_data_ptr());
inputs.push_back(tv->get_data_ptr());
}
for (size_t i = 0; i < output_tvs.size(); i++)
{
shared_ptr<runtime::cpu::CPUTensorView> tv =
static_pointer_cast<runtime::cpu::CPUTensorView>(output_tvs[i]);
m_outputs.push_back(tv->get_data_ptr());
outputs.push_back(tv->get_data_ptr());
}
// Invoke compiled computation
m_compiled_function(this);
m_compiled_function(inputs, outputs);
}
void CallFrame::operator()(const std::vector<std::shared_ptr<ngraph::runtime::Value>>& arguments,
......@@ -68,15 +66,3 @@ void CallFrame::operator()(const std::vector<std::shared_ptr<ngraph::runtime::Va
tensor_call(inputs, outputs);
}
void* CallFrame::get_input_data(size_t index)
{
void* rc = m_inputs.at(index);
return rc;
}
void* CallFrame::get_output_data(size_t index)
{
void* rc = m_outputs.at(index);
return rc;
}
......@@ -32,7 +32,8 @@ namespace ngraph
{
class CallFrame;
using EntryPoint_t = void(ngraph::runtime::cpu::CallFrame* call_frame);
using EntryPoint_t = void(const std::vector<void*>& inputs,
const std::vector<void*>& outputs);
using EntryPoint = std::function<EntryPoint_t>;
......@@ -40,8 +41,7 @@ namespace ngraph
class CallFrame : public ngraph::runtime::CallFrame
{
public:
CallFrame(EntryPoint compiled_function,
const std::vector<std::shared_ptr<CallFrame>>& callees);
CallFrame(EntryPoint compiled_function);
/// @brief Invoke the function with values matching the signature of the function.
///
......@@ -56,19 +56,9 @@ namespace ngraph
const std::vector<std::shared_ptr<TensorView>>& outputs);
void set_return() { m_return = true; }
// const std::vector<std::shared_ptr<ngraph::runtime::Value>>& get_inputs();
// const std::vector<std::shared_ptr<ngraph::runtime::Value>>& get_outputs();
void* get_input_data(size_t index);
void* get_output_data(size_t index);
protected:
bool m_return;
EntryPoint m_compiled_function;
std::vector<std::shared_ptr<CallFrame>> m_callees;
std::vector<void*> m_inputs;
std::vector<void*> m_outputs;
};
}
}
......
This diff is collapsed.
......@@ -25,7 +25,6 @@
#define EMITTER_DECL(E) \
E(const ngraph::Node* n, \
ExternalFunction* ef, \
FunctionMap& function_map, \
const std::vector<TensorViewInfo>& inputs, \
const std::vector<TensorViewInfo>& outputs)
......
......@@ -42,7 +42,6 @@ namespace ngraph
using OpFunction = std::function<void(Emitter*,
const ngraph::Node*,
ExternalFunction*,
FunctionMap&,
const std::vector<TensorViewInfo>& inputs,
const std::vector<TensorViewInfo>& outputs)>;
......@@ -54,15 +53,11 @@ namespace ngraph
ExternalFunction(const std::shared_ptr<ngraph::Function>& function,
bool release_function = true);
std::shared_ptr<ngraph::runtime::CallFrame> make_call_frame();
std::vector<std::shared_ptr<CallFrame>>& get_callees() { return callees; }
protected:
void compile(FunctionMap& function_map);
void compile();
size_t m_n_inputs;
size_t m_n_outputs;
ngraph::descriptor::TensorViewPtrs m_temp_views;
EntryPoint m_compiled_function;
std::vector<std::shared_ptr<CallFrame>> callees;
};
}
}
......
......@@ -1422,10 +1422,10 @@ TEST(${BACKEND_NAME}, reduce_trivial)
auto cf = backend->make_call_frame(external);
// Create some tensors for input/output
auto a = runtime::make_tensor<element::Float32>(shape);
auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
copy_data(a, vector<float>{1, 2, 3, 4});
auto b = runtime::make_tensor<element::Float32>(shape);
copy_data(b, vector<float>{0});
auto b = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
copy_data(b, vector<float>{0, 0, 0, 0});
auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
(*cf)({a, b}, {result});
......
......@@ -113,3 +113,24 @@ TEST(tensor, read_write)
test_read_write<element::Float32>({1.0, 3.0, 5.0});
test_read_write<element::Int64>({-1, 2, 4});
}
TEST(tensor, output_flag)
{
pass::Manager pass_manager;
pass_manager.register_pass<pass::TopologicalSort>();
pass_manager.register_pass<pass::Liveness>();
auto arg0 = make_shared<op::Parameter>(element::Float32::element_type(), Shape{1});
auto add = make_shared<op::Add>(arg0, arg0);
auto rt = make_shared<TensorViewType>(element::Float32::element_type(), Shape{1});
auto f0 = make_shared<Function>(add, rt, op::Parameters{arg0});
pass_manager.run_passes(f0);
EXPECT_TRUE(f0->get_result()->is_output());
for (descriptor::Output& output : f0->get_result()->get_outputs())
{
const Tensor& t = output.get_tensor();
EXPECT_TRUE(t.is_output());
}
}
......@@ -21,7 +21,6 @@
#include "ngraph/log.hpp"
#include "ngraph/ngraph.hpp"
#include "ngraph/pass/collect_functions.hpp"
#include "ngraph/pass/dump_sorted.hpp"
#include "ngraph/pass/manager.hpp"
#include "ngraph/pass/topological_sort.hpp"
......@@ -172,11 +171,11 @@ TEST(topological_sort, collect_functions)
"h");
pass::Manager pass_manager;
pass_manager.register_pass<pass::CollectFunctions>();
pass_manager.run_passes(h);
set<string> expected = {"f", "g", "h"};
auto functions = pass_manager.get_state().get_functions();
vector<string> fnames;
for (shared_ptr<Function> func : functions)
{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment