Merge branch 'master' into krovatkin/min_gcc_ver

.gitignore

@@ -69,3 +69,7 @@ config_args.txt
 .nfs*
 venv/
 .vscode/
+
+# VisualGDB files
+VisualGDB/
+toolchain.cmake

README.md

@@ -12,6 +12,8 @@ TODO
 
 ## Steps
 
+_If you are developing ngraph on macOS (officially unsupported) please see the section "macOS Development Prerequisites" below._
+
 `libngraph` is build in the customary manner for a CMake-based project:
 
 1. Create a build directory outside of source directory tree.
@@ -23,6 +25,17 @@ TODO
 6. _(Optional, requires `doxygen`)_ Run `make doc`.
     * This will build API documentation in the directory `doc` inside the build directory.
 
+## macOS Development Prerequisites
+
+The repository includes two scripts (`maint/check-code-format.sh` and `maint/apply-code-format.sh`) that are used respectively to check adherence to `libngraph` code formatting conventions, and to automatically reformat code according to those conventions. These scripts require the command `clang-format-3.9` to be in your `PATH`. Run the following commands (you will need to adjust them if you are not using `bash`).
+
+```
+$ brew install llvm@3.9
+$ mkdir -p $HOME/bin
+$ ln -s /usr/local/opt/llvm@3.9/bin/clang-format $HOME/bin/clang-format-3.9
+$ echo 'export PATH=$HOME/bin:$PATH' >> $HOME/.bash_profile
+```
+
 # Testing `libngraph`
 
 `libngraph` uses the GTest framework for unit tests.   CMake automatically downloads a

cmake/clang_4_0_flags.cmake

@@ -13,6 +13,7 @@
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=inconsistent-missing-override")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic-errors")
 
  # whitelist errors here
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything")

cmake/external_eigen.cmake

@@ -15,6 +15,9 @@
 include(ExternalProject)
 
 set(EIGEN_INSTALL_DIR ${EXTERNAL_INSTALL_DIR}/eigen)
+set(EIGEN_PROJECT eigen)
+set(EIGEN_SHA1_HASH dd238ca6c6b5d2ce2e7e2e9ded4c59bad77ce6d0)
+set(EIGEN_URL http://bitbucket.org/eigen/eigen/get/3.3.3.zip)
 
 #----------------------------------------------------------------------------------------------------------
 # Download and install GoogleTest ...
@@ -23,17 +26,17 @@ set(EIGEN_INSTALL_DIR ${EXTERNAL_INSTALL_DIR}/eigen)
 # The 'BUILD_BYPRODUCTS' argument was introduced in CMake 3.2.
 if (${CMAKE_VERSION} VERSION_LESS 3.2)
     ExternalProject_Add(
-        eigen
-        URL http://bitbucket.org/eigen/eigen/get/3.3.3.zip
-        # PREFIX ${CMAKE_CURRENT_BINARY_DIR}/eigen
+        ${EIGEN_PROJECT}
+        URL ${EIGEN_URL}
+        URL_HASH SHA1=${EIGEN_SHA1_HASH}
         UPDATE_COMMAND ""
         CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EIGEN_INSTALL_DIR}
         )
 else()
     ExternalProject_Add(
-        eigen
-        URL http://bitbucket.org/eigen/eigen/get/3.3.3.zip
-        # PREFIX ${CMAKE_CURRENT_BINARY_DIR}/eigen
+        ${EIGEN_PROJECT}
+        URL ${EIGEN_URL}
+        URL_HASH SHA1=${EIGEN_SHA1_HASH}
         UPDATE_COMMAND ""
         CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EIGEN_INSTALL_DIR}
         BUILD_BYPRODUCTS "${EIGEN_INSTALL_DIR}/include/eigen3"

contrib/docker/Dockerfile.ngraph_cpp_cpu

@@ -11,7 +11,4 @@ RUN apt-get update && apt-get install -y \
 RUN apt-get clean autoclean && \
     apt-get autoremove -y
 
-# Add chown_files script to avoid 
-ADD contrib/docker/chown_files.sh /tmp/chown_files.sh
-
-WORKDIR /root/ngraph-cpp-test
+WORKDIR /home

contrib/docker/Makefile

 # Basic Makefile for contrib/docker. This can be expanded later as more targets
 # are added.
 
+# Default is to build with -j for parallel builds.  Turn off with
+#   make PARELLEL=
+PARALLEL=-j
+
+# DIR is an internal variable that serves as an anchor to this cloned git
+# repository.  DIR is mounted into the docker container, so that builds
+# can occur within the container on this cloned git repository.  DIR should
+# not be modified - if it is, then the build system will not work.
 DIR = $(realpath ../..)
-# Use /tmp/ngraph-cpp-test, because we run as the user (and not root)
+
+# DOCKUSER_HOME is the location of the home directory of the fabricated
+# "dockuser" user, used only within the docker containers.  "dockuser" is
+# created (from the passed-in RUN_UID) to map the docker-caller user's UID to a
+# first-class user (/etc/passwd entry, member of sudo group, proper home dir)
+# /home/dockuser is also used in other scripts, notably run_as_user.sh, so if
+# changed it must be done in other areas for the builds to work.
+DOCKUSER_HOME=/home/dockuser
+
+# Use /home/dockuser/ngraph-cpp-test, because we run as the user (and not root)
 # /root/ngraph-cpp-test is not used, because /root is not accessible to user
-VOLUME = -v ${DIR}:/tmp/ngraph-cpp-test
+VOLUME = -v "${DIR}:${DOCKUSER_HOME}/ngraph-cpp-test"
 GIT_COMMIT = $(shell git rev-parse HEAD)
 BUILD_VERSION = ${GIT_COMMIT}_${PYTHON_VERSION}
 BUILD_DIR = ${DIR}/contrib/docker/.build-${BUILD_VERSION}
@@ -16,56 +33,71 @@ CALLER_GID := $(shell id -g)
 # line
 PYTHON_VERSION = 2
 
-.PHONY: clean build_ngraph_cpp_cpu check_cpu shell build_all
+.PHONY: clean build_ngraph_cpp_cpu check_cpu install shell build_all
 
 DOCKER_BUILD=docker build --rm=true
 
 ifdef http_proxy
 DOCKER_BUILD+=--build-arg http_proxy=$(http_proxy)
-DOCKER_RUN_ENV+=--env http_proxy=$(http_proxy)
+DOCKER_RUN_ENV+=--env "http_proxy=$(http_proxy)"
 endif
 
 ifdef https_proxy
 DOCKER_BUILD+=--build-arg https_proxy=$(https_proxy)
-DOCKER_RUN_ENV+=--env https_proxy=$(https_proxy)
+DOCKER_RUN_ENV+=--env "https_proxy=$(https_proxy)"
 endif
 
 expand_dockerfile_templates:
-	cd ${DIR}/contrib/docker
-	mkdir ${BUILD_DIR} || true
-	sed -e 's/\(FROM ngraph.*\)/\1:${BUILD_VERSION}/' Dockerfile.ngraph_cpp_cpu > ${BUILD_DIR}/Dockerfile.ngraph_cpp_cpu
+	cd "${DIR}"/contrib/docker
+	mkdir "${BUILD_DIR}" || true
+	sed -e 's/\(FROM ngraph.*\)/\1:${BUILD_VERSION}/' Dockerfile.ngraph_cpp_cpu > "${BUILD_DIR}"/Dockerfile.ngraph_cpp_cpu
 
 clean:
-	rm -f ${DIR}/contrib/docker/.build-*/Dockerfile.* || echo "keep going if files are not present"
-	rmdir ${DIR}/contrib/docker/.build-* || echo "keep going if directory is not present"
+	rm -f "${DIR}"/contrib/docker/.build-*/Dockerfile.* || echo "keep going if files are not present"
+	rmdir "${DIR}"/contrib/docker/.build-* || echo "keep going if directory is not present"
+	rm -fr "${DIR}"/BUILD
 
 build_ngraph_cpp_cpu: expand_dockerfile_templates
-	$(DOCKER_BUILD) -f=${BUILD_DIR}/Dockerfile.ngraph_cpp_cpu --build-arg python_version=${PYTHON_VERSION} -t=ngraph_cpp_cpu:${BUILD_VERSION} ${DIR}
+	$(DOCKER_BUILD) -f="${BUILD_DIR}"/Dockerfile.ngraph_cpp_cpu --build-arg python_version="${PYTHON_VERSION}" -t=ngraph_cpp_cpu:"${BUILD_VERSION}" "${DIR}"
 	# remove the tag for the previous latest image
 	docker rmi ngraph_cpp_cpu:latest || echo "keep going if docker rmi command fails"
-	docker tag `docker images -q ngraph_cpp_cpu:${BUILD_VERSION}` ngraph_cpp_cpu:latest
+	docker tag `docker images -q "ngraph_cpp_cpu:${BUILD_VERSION}"` ngraph_cpp_cpu:latest
 
 build_all: build_ngraph_cpp_cpu
 
 check_cpu: build_ngraph_cpp_cpu
+	# Remove old distribution directory if present
+	( test -d "${DIR}"/BUILD/ngraph_dist && rm -fr "${DIR}"/BUILD/ngraph_dist && echo "Removed old ${DIR}/BUILD/ngraph_dist directory" ) || echo "Previous ngraph_dist directory not found"
 	# Make BUILD directory as user
-	mkdir -p ${DIR}/BUILD
-	chmod ug+rwx ${DIR}/BUILD
-	# Need to use /tmp/ngraph-cpp-test/BUILD, because running as user
-	# Can't use /root/ngraph-cpp-test/BUILD, because /root not accessible to user
-	docker run \
-	    --rm --user ${CALLER_UID}:${CALLER_GID} \
-	    ${DOCKER_RUN_ENV} ${VOLUME} -w /tmp/ngraph-cpp-test/BUILD -t ngraph_cpp_cpu:${BUILD_VERSION} \
-	    sh -c "cmake -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 .. ; env VERBOSE=1 make check"
-	# update the files to be owned by the calling user instead of root, to avoid docker mount problems with file ownership
-	docker run --rm ${VOLUME} \
-	--env MY_UID=${CALLER_UID} \
-	--env MY_GID=${CALLER_GID} \
-	--env MY_ROOT_DIR=/root/ngraph-cpp-test \
-	-t ngraph_cpp_cpu \
-	/tmp/chown_files.sh
+	mkdir -p "${DIR}"/BUILD
+	chmod ug+rwx "${DIR}"/BUILD
+	docker run --rm --tty \
+            ${VOLUME} \
+	    ${DOCKER_RUN_ENV} \
+            --env RUN_UID="$(shell id -u)" \
+            --env RUN_CMD="set -e ; set -o pipefail ; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD; cmake -DCMAKE_CXX_COMPILER=clang++-3.9 -DCMAKE_C_COMPILER=clang-3.9 .. 2>&1 | tee cmake.log ; env VERBOSE=1 make ${PARALLEL} 2>&1 | tee make.log ; env VERBOSE=1 make check 2>&1 | tee make_check.log" \
+            "ngraph_cpp_cpu:${BUILD_VERSION}" \
+	    sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
 
 shell: build_ngraph_cpp_cpu
-	docker run --rm ${VOLUME} -it ngraph_cpp_cpu:${BUILD_VERSION} /bin/bash
+	# "make shell" runs an interactive shell in the docker image, for debugging
+	docker run --rm --tty --interactive \
+            ${VOLUME} \
+	    ${DOCKER_RUN_ENV} \
+            --env RUN_UID="$(shell id -u)" \
+            "ngraph_cpp_cpu:${BUILD_VERSION}" \
+            sh -c "cd ${DOCKUSER_HOME} ; ${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
+
+install:
+	# Puts ngraph_dist in BUILD directory.  This is used by Jenkins ngraph-tensorflow batch job.
+	# Note: We currently have a bug where cmake only installs in $HOME.  Jira NGTF-205 is opened
+	#       for this.  For now, here we install to $HOME, then move the directory.
+	docker run --rm --tty \
+            ${VOLUME} \
+	    ${DOCKER_RUN_ENV} \
+            --env RUN_UID="$(shell id -u)" \
+            --env RUN_CMD="set -e ; set -o pipefail; cd ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD ; test -d ngraph_dist && rm -fr ngraph_dist && echo 'Removed old ngraph_dist directory' ; make install 2>&1 | tee make_install.log ; mv -v ${DOCKUSER_HOME}/ngraph_dist ${DOCKUSER_HOME}/ngraph-cpp-test/BUILD" \
+            "ngraph_cpp_cpu:${BUILD_VERSION}" \
+	    sh -c "${DOCKUSER_HOME}/ngraph-cpp-test/contrib/docker/run_as_user.sh"
 
 all: build_ngraph_cpp_cpu

contrib/docker/chown_files.sh

-#!/bin/bash
-
-# the docker run commands leave output files with root ownership
-# modify the file ownership with the UID of the calling user  
-
-if [ -z $MY_UID ];then
-    MY_UID=`id -u`
-fi
-
-if [ -z $MY_GID ];then
-    MY_GID=`id -g`
-fi
-
-if [ -z $MY_ROOT_DIR ];then
-    MY_ROOT_DIR=/root/ngraph-test
-fi
-
-cd $MY_ROOT_DIR
-find . -user root > files_to_chown.txt
-cat files_to_chown.txt | xargs chown ${MY_UID} ${1}
-cat files_to_chown.txt | xargs chgrp ${MY_GID} ${1}
-rm files_to_chown.txt

contrib/docker/docker_cleanup.sh

@@ -7,7 +7,7 @@ echo
 
 # clean up old docker containers
 echo "Removing Exited docker containers..." 
-docker ps -a | grep Exited | cut -f 1 -d ' ' | xargs docker rm -f ${1}
+docker ps -a | grep Exited | cut -f 1 -d ' ' | xargs docker rm -f "${1}"
 echo
 
 #list docker images for ngraph
@@ -17,4 +17,4 @@ echo
 
 # clean up docker images no longer in use
 echo "Removing docker images for ngraph..." 
-docker images -qa ngraph_* | xargs docker rmi -f ${1}
+docker images -qa ngraph_* | xargs docker rmi -f "${1}"

contrib/docker/run_as_user.sh

+#!  /bin/bash
+
+# This script is designed to simulate running as a user with a particular UID
+# within a docker container.
+#
+# Normally a docker container runs as root, which can cause problems with file
+# ownership when a host directory tree is mounted into the docker container.
+# There are other problems with building and running software as root as
+# well.  Good practice when validating software builds in a docker container
+# is to run as a normal user, since many (most?) end users will not be building
+# and installing software as root.
+#
+# This script should be run using "docker run", with RUN_UID (set to the user
+# you want to run as) passed into the docker container as an environment
+# variable.  The script will then add the UID as user "dockuser" to
+# /etc/passwd (important for some software, like bazel), add the new dockuser
+# to the sudo group (whether or not sudo is installed), and su to a new shell
+# as the dockuser (passing in the existing environment, which is important).
+#
+# If the environment variable RUN_CMD is passed into the docker container, then
+# this script will use RUN_CMD as a command to run when su'ing.  If RUN_CMD is
+# not defined, then /bin/bash will run, which effectively provides an
+# interactive shell in the docker container, for debugging.
+
+set -e  # Make sure we exit on any command that returns non-zero
+set -u  # No unset variables
+
+if [ -z "$RUN_UID" ] ; then
+
+    # >&2 redirects echo output to stderr.
+    # See: https://stackoverflow.com/questions/2990414/echo-that-outputs-to-stderr
+    ( >&2 echo 'ERROR: Environment variable RUN_UID was not set when run-as-user.sh was run' )
+    ( >&2 echo '       Running as default user (root, in docker)' )
+    ( >&2 echo ' ' )
+
+    exit 1
+
+else
+
+    # The username used in the docker container to map the caller UID to
+    #
+    # Note 'dockuser' is used in other scripts, notably Makefile.  If you
+    # choose to change it here, then you need to change it in all other
+    # scripts, or else the builds will break.
+    #
+    DOCK_USER='dockuser'
+
+    # We will be su'ing using a non-login shell or command, and preserving
+    # the environment.  This is done so that env. variables passed in with
+    # "docker run --env ..." are honored.
+    # Therefore, we need to reset at least HOME=/root ...
+    #
+    # Note also that /home/dockuser is used in other scripts, notably
+    # Makefile.  If you choose to change it here, then you need to change it
+    # in all other scripts, or else the builds will break.
+    #
+    export HOME="/home/${DOCK_USER}"
+
+    # Make sure the home directory is owned by the new user
+    if [ -d "${HOME}" ] ; then
+      chown "${RUN_UID}" "${HOME}"
+    fi
+
+    # Add a user with UID of person running docker (in ${RUN_UID})
+    # If $HOME does not yet exist, then it will be created
+    adduser --disabled-password --gecos 'Docker-User' -u "${RUN_UID}" "${DOCK_USER}"
+    # Add dockuser to the sudo group
+    adduser "${DOCK_USER}" sudo
+
+    # If root access is needed in the docker image while running as a normal
+    # user, uncomment this and add 'sudo' as a package installed in Dockerfile
+    # echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+    if [ -z "${RUN_CMD+x}" ] ; then  # Launch a shell as dockuser
+      su -m "${DOCK_USER}" -c /bin/bash
+    else                         # Run command as dockuser
+      su -m "${DOCK_USER}" -c "${RUN_CMD}"
+    fi
+
+fi

doc/Doxyfile.in

@@ -3,5 +3,8 @@ PROJECT_BRIEF    = "Nervana graph compiler"
 
 OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@
 INPUT            = @CMAKE_SOURCE_DIR@/src
-
 RECURSIVE        = YES
+
+EXTRACT_STATIC   = YES
+
+USE_MATHJAX      = YES

src/ngraph/CMakeLists.txt

@@ -12,6 +12,10 @@
 # limitations under the License.
 
 set (SRC
+    autodiff/adjoints.cpp
+    autodiff/backprop_derivative.cpp
+    autodiff/backprop_function.cpp
+    autodiff/numeric_derivative.cpp
     descriptor/input.cpp
     descriptor/layout/dense_tensor_view_layout.cpp
     descriptor/layout/tensor_view_layout.cpp
@@ -23,6 +27,7 @@ set (SRC
     function.cpp
     log.cpp
     node.cpp
+    ops/add.cpp
     ops/binary_elementwise_arithmetic.cpp
     ops/binary_elementwise_builtin.cpp
     ops/binary_elementwise_comparison.cpp
@@ -30,15 +35,22 @@ set (SRC
     ops/concatenate.cpp
     ops/constant.cpp
     ops/convert.cpp
+    ops/divide.cpp
     ops/dot.cpp
+    ops/exp.cpp
     ops/function_call.cpp
     ops/get_tuple_element.cpp
+    ops/log.cpp
+    ops/multiply.cpp
+    ops/negative.cpp
     ops/op.cpp
     ops/parameter.cpp
     ops/reduce.cpp
     ops/reshape.cpp
     ops/select.cpp
     ops/slice.cpp
+    ops/subtract.cpp
+    ops/sum.cpp
     ops/tuple.cpp
     ops/unary_elementwise_arithmetic.cpp
     ops/unary_elementwise_builtin.cpp
@@ -63,6 +75,7 @@ set (SRC
     runtime/tensor_view.cpp
     runtime/tuple.cpp
     runtime/utils.cpp
+    test/all_close.cpp
     shape.cpp
     types/element_type.cpp
     types/type.cpp

src/ngraph/autodiff/adjoints.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cassert>
+#include <list>
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/broadcast.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+/// @brief Make a zero matching a value type.
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const ValueType>& value_type);
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const TensorViewType>& tensor_view_type)
+{
+    std::shared_ptr<Node> zero =
+        std::make_shared<op::Constant>(tensor_view_type->get_element_type(), Shape{}, "0");
+    const Shape& shape = tensor_view_type->get_shape();
+    if (shape.size() > 0)
+    {
+        AxisSet axes;
+        for (size_t i = 0; i < shape.size(); i++)
+        {
+            axes.insert(i);
+        }
+        zero = std::make_shared<op::Broadcast>(zero, shape, axes);
+    }
+    return zero;
+}
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const TupleType>& tuple_type)
+{
+    std::vector<std::shared_ptr<Node>> elements;
+    for (auto& value_type : tuple_type->get_element_types())
+    {
+        elements.push_back(make_zero(value_type));
+    }
+    return std::make_shared<op::Tuple>(elements);
+}
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const ValueType>& value_type)
+{
+    std::shared_ptr<const TensorViewType> tensor_view_type =
+        std::dynamic_pointer_cast<const TensorViewType>(value_type);
+    if (nullptr != tensor_view_type)
+    {
+        return (make_zero(tensor_view_type));
+    }
+    std::shared_ptr<const TupleType> tuple_type =
+        std::dynamic_pointer_cast<const TupleType>(value_type);
+    if (nullptr != tuple_type)
+    {
+        return make_zero(tuple_type);
+    }
+    // Should be impossible
+    throw ngraph_error("Unknown value type");
+}
+
+autodiff::Adjoints::Adjoints(const std::shared_ptr<Node>& y, const std::shared_ptr<Node>& c)
+{
+    // Pass 1 determines which nodes contribute to y as well as setting up a reverse
+    // topological sort.
+
+    // Number of nodes that use the node's value
+    std::unordered_map<std::shared_ptr<Node>, size_t> parent_counts;
+
+    // Nodes that have been processed
+    std::unordered_set<std::shared_ptr<Node>> visited_nodes;
+
+    // Nodes we should check
+    std::list<std::shared_ptr<Node>> nodes_to_check;
+    nodes_to_check.push_front(y);
+    while (nodes_to_check.size() > 0)
+    {
+        auto node = nodes_to_check.front();
+        nodes_to_check.pop_front();
+        if (visited_nodes.count(node) != 0)
+        {
+            continue;
+        }
+        for (auto arg : node->get_arguments())
+        {
+            auto count_it = parent_counts.find(arg);
+            if (count_it == parent_counts.end())
+            {
+                parent_counts[arg] = 1;
+                nodes_to_check.push_front(arg);
+            }
+            else
+            {
+                parent_counts[arg]++;
+            }
+        }
+        visited_nodes.insert(node);
+    }
+
+    // Second pass visits the nodes so that all users of a node's value are visited
+    // before a node is visited.
+    m_adjoint_map[y.get()] = c;
+    nodes_to_check.push_front(y);
+    while (nodes_to_check.size() > 0)
+    {
+        auto node = nodes_to_check.front();
+        nodes_to_check.pop_front();
+        // Look for nodes that will be available when this node is done
+        for (auto arg : node->get_arguments())
+        {
+            auto count_it = parent_counts.find(arg);
+            count_it->second--;
+            if (0 == count_it->second)
+            {
+                nodes_to_check.push_front(arg);
+            }
+        }
+        node->generate_adjoints(*this, m_adjoint_map.at(node.get()));
+    }
+}
+
+std::shared_ptr<Node> autodiff::Adjoints::get(const std::shared_ptr<Node>& x)
+{
+    auto adjoint_it = m_adjoint_map.find(x.get());
+    if (m_adjoint_map.end() == adjoint_it)
+    {
+        auto result = make_zero(x->get_value_type());
+        adjoint_it = m_adjoint_map.insert({x.get(), result}).first;
+    }
+    return adjoint_it->second;
+}
+
+void autodiff::Adjoints::add_delta(const std::shared_ptr<Node>& x,
+                                   const std::shared_ptr<Node>& delta)
+{
+    assert(*x->get_value_type() == *delta->get_value_type());
+    auto adjoint_it = m_adjoint_map.find(x.get());
+    if (m_adjoint_map.end() == adjoint_it)
+    {
+        m_adjoint_map.insert({x.get(), delta});
+    }
+    else
+    {
+        m_adjoint_map.insert({x.get(), std::make_shared<op::Add>(adjoint_it->second, delta)});
+    }
+}

src/ngraph/autodiff/adjoints.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        class Adjoints
+        {
+        public:
+            /// @brief (dy/dx)(c) for all x used to compute y
+            ///
+            /// @param y The dependent value
+            /// @param c An expression for where to evaluate the derivatives
+            Adjoints(const std::shared_ptr<Node>& y, const std::shared_ptr<Node>& c);
+
+            Adjoints(const Adjoints& adjoints) = default;
+            Adjoints& operator=(const Adjoints& adjoints) = default;
+            Adjoints() = default;
+
+            /// @brief (dy/dx)(c)
+            ///
+            /// @param x The node whose adjoint is desired.
+            std::shared_ptr<Node> get(const std::shared_ptr<Node>& x);
+
+            /// @brief Add a backprop contribution to x's adjoint
+            ///
+            /// @param x The adjoint node
+            /// @param delta A backprop contribution
+            void add_delta(const std::shared_ptr<Node>& x, const std::shared_ptr<Node>& delta);
+
+        protected:
+            std::unordered_map<Node*, std::shared_ptr<Node>> m_adjoint_map;
+        };
+
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/backprop_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/autodiff/backprop_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::backprop_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args)
+{
+    auto y = f->get_result();
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    auto c_param = std::make_shared<op::Parameter>(ET::element_type(), y_shape);
+    auto c_arg = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto params = f->get_parameters();
+
+    std::vector<std::shared_ptr<Node>> deriv_nodes;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> bprops;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (auto param : params)
+    {
+        Shape s = y_shape;
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+        bprops.push_back(backend->make_parameterized_tensor_view<ET>(param_shape));
+        deriv_nodes.push_back(y->backprop_node(param, c_param));
+    }
+
+    std::vector<std::shared_ptr<op::Parameter>> df_params = params;
+    df_params.push_back(c_param);
+    auto df_result = std::make_shared<op::Tuple>(deriv_nodes);
+    auto df = std::make_shared<Function>(df_result, df_result->get_value_type(), df_params);
+
+    auto external = manager->compile(df);
+    auto cf = backend->make_call_frame(external);
+
+    // We compute the derivatives chunk by chunk
+    std::vector<typename std::vector<typename ET::type>::iterator> result_pos;
+    for (auto result : results)
+    {
+        result_pos.push_back(result->get_vector().begin());
+    }
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+    args_tv.push_back(c_arg);
+
+    runtime::TensorViewPtrs bprops_tv;
+    bprops_tv.insert(bprops_tv.begin(), bprops.begin(), bprops.end());
+
+    auto& c_vec = c_arg->get_vector();
+    for (size_t i = 0; i < c_vec.size(); i++)
+    {
+        c_vec[i] = 1;
+        cf->tensor_call(args_tv, bprops_tv);
+        c_vec[i] = 0;
+        for (size_t j = 0; j < results.size(); j++)
+        {
+            auto& bprop_vec = bprops[j]->get_vector();
+            result_pos[j] = std::copy(bprop_vec.begin(), bprop_vec.end(), result_pos[j]);
+        }
+    }
+
+    return results;
+}
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+    autodiff::backprop_derivative<ngraph::element::Float32>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+    autodiff::backprop_derivative<ngraph::element::Float64>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>& args);

src/ngraph/autodiff/backprop_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/backprop_function.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cassert>
+#include <list>
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/broadcast.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+std::shared_ptr<Function> autodiff::backprop_function(const std::shared_ptr<Function>& f)
+{
+    auto Y = f->get_result();
+    auto Xs = f->get_parameters();
+    auto C = std::make_shared<op::Parameter>(Y->get_value_type());
+    std::vector<std::shared_ptr<Node>> dYdXs(Xs.size());
+    transform(Xs.begin(), Xs.end(), dYdXs.begin(), [C, Y](const std::shared_ptr<Node>& X) {
+        return Y->backprop_node(X, C);
+    });
+    auto result = std::make_shared<op::Tuple>(dYdXs);
+    std::vector<std::shared_ptr<op::Parameter>> params(Xs);
+    params.push_back(C);
+    return std::make_shared<Function>(result, result->get_value_type(), params);
+}

src/ngraph/autodiff/backprop_function.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/numeric_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+
+#include "ngraph/autodiff/numeric_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+        typename ET::type delta)
+{
+    auto y = f->get_result();
+
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    auto params = f->get_parameters();
+
+    // Results for each derivative, shape Y|X_i
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (auto param : params)
+    {
+        Shape s = y_shape;
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+    }
+
+    auto external = manager->compile(f);
+    auto cf = backend->make_call_frame(external);
+
+    // ref_y is the function evaluated at the args
+    auto ref_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+
+    cf->tensor_call(args_tv, runtime::TensorViewPtrs{ref_y});
+    auto& ref_vec = ref_y->get_vector();
+
+    // inc_y will hold f(x+dx) values
+    auto inc_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto& inc_vec = inc_y->get_vector();
+
+    // Assuming vars, y, and results are row-major
+
+    typename ET::type inv_delta = 1 / delta;
+    for (size_t i = 0; i < args.size(); ++i)
+    {
+        auto arg = args[i];
+        auto df_darg = results[i];
+        auto df_darg_it = df_darg->get_vector().begin();
+        auto& vec = arg->get_vector();
+        for (size_t j = 0; j < vec.size(); j++)
+        {
+            auto old_val = vec[j];
+            vec[j] += delta;
+            cf->tensor_call(args_tv, {inc_y});
+            vec[j] = old_val;
+            df_darg_it = std::transform(inc_vec.begin(),
+                                        inc_vec.end(),
+                                        ref_vec.begin(),
+                                        df_darg_it,
+                                        [inv_delta](typename ET::type y1, typename ET::type y0) {
+                                            return inv_delta * (y1 - y0);
+                                        });
+        }
+    }
+    return results;
+}
+
+template std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>&
+            args,
+        element::Float32::type delta);
+
+template std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>&
+            args,
+        element::Float64::type delta);

src/ngraph/autodiff/numeric_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/runtime/tuple.hpp"
+#include "ngraph/runtime/value.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace autodiff
+    {
+        /// @brief numeric approximation of the derivative
+        /// @param f A function
+        /// @param args Values for the arguments (the independent variables)
+        /// @param delta increment for the variables
+        /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> numeric_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+            typename ET::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args,
+                element::Float32::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args,
+                element::Float64::type delta);
+    }
+}

src/ngraph/descriptor/layout/dense_tensor_view_layout.cpp

@@ -44,3 +44,21 @@ size_t DenseTensorViewLayout::get_index_offset(const std::vector<size_t>& indice
     }
     return result;
 }
+
+bool DenseTensorViewLayout::operator==(const TensorViewLayout& other) const
+{
+    const DenseTensorViewLayout* p_other = dynamic_cast<const DenseTensorViewLayout*>(&other);
+    if (nullptr == p_other)
+        return false;
+
+    if (get_element_type() != p_other->get_element_type())
+        return false;
+
+    if (m_strides != p_other->m_strides)
+        return false;
+
+    if (m_offset != p_other->m_offset)
+        return false;
+
+    return true;
+}

src/ngraph/descriptor/layout/dense_tensor_view_layout.hpp

@@ -41,9 +41,11 @@ namespace ngraph
                 virtual size_t get_index_offset(const std::vector<size_t>& indices) override;
 
                 const Strides& get_strides() const { return m_strides; }
+                virtual bool operator==(const TensorViewLayout& other) const override;
+
             protected:
                 Strides m_strides;
-                size_t m_offset;
+                size_t m_offset{0};
                 size_t m_size;
             };
         }

src/ngraph/descriptor/layout/tensor_view_layout.hpp

@@ -59,6 +59,9 @@ namespace ngraph
                 /// Where this view is located in the buffer.
                 const BufferPos& get_buffer_pos() const { return m_buffer_pos; }
                 BufferPos& get_buffer_pos() { return m_buffer_pos; }
+                /// @brief Return true if this and other have the same element interpretation
+                virtual bool operator==(const TensorViewLayout& other) const = 0;
+                bool operator!=(const TensorViewLayout& other) const { return !(*this == other); }
             protected:
                 std::shared_ptr<const TensorViewType> m_tensor_view_type;
                 BufferPos m_buffer_pos;

src/ngraph/function.cpp

@@ -23,7 +23,7 @@ using namespace ngraph;
 atomic<size_t> Function::m_next_instance_id(0);
 
 Function::Function(const std::shared_ptr<Node>& result,
-                   const std::shared_ptr<ValueType>& result_type,
+                   const std::shared_ptr<const ValueType>& result_type,
                    const std::vector<std::shared_ptr<op::Parameter>>& parameters,
                    const std::string& name)
     : m_result(result)
@@ -33,32 +33,26 @@ Function::Function(const std::shared_ptr<Node>& result,
     , m_ordered_ops_valid(false)
     , m_instance_id(m_next_instance_id.fetch_add(1))
 {
-    size_t i = 0;
-    for (auto parameter : parameters)
-    {
-        parameter->assign_function(this, i++);
-    }
-
-    traverse_nodes(result, [&](Node* node) { m_ops.push_back(node); });
+    traverse_nodes(this, [&](shared_ptr<Node> node) { m_ops.push_back(node); });
 }
 
-void Function::set_ordered_ops(const std::list<Node*>& ordered_ops)
+void Function::set_ordered_ops(const std::list<shared_ptr<Node>>& ordered_ops)
 {
     m_ordered_ops = ordered_ops;
     m_ordered_ops_valid = true;
 }
 
-std::list<Node*>& Function::get_ops()
+std::list<shared_ptr<Node>>& Function::get_ops()
 {
     return m_ops;
 }
 
-const std::list<Node*>& Function::get_ops() const
+const std::list<shared_ptr<Node>>& Function::get_ops() const
 {
     return m_ops;
 }
 
-std::list<Node*>& Function::get_ordered_ops()
+std::list<shared_ptr<Node>>& Function::get_ordered_ops()
 {
     if (!m_ordered_ops_valid)
     {
@@ -67,7 +61,7 @@ std::list<Node*>& Function::get_ordered_ops()
     return m_ordered_ops;
 }
 
-const std::list<Node*>& Function::get_ordered_ops() const
+const std::list<shared_ptr<Node>>& Function::get_ordered_ops() const
 {
     if (!m_ordered_ops_valid)
     {

src/ngraph/function.hpp

@@ -35,23 +35,23 @@ namespace ngraph
     {
     public:
         Function(const std::shared_ptr<Node>& result,
-                 const std::shared_ptr<ValueType>& result_type,
+                 const std::shared_ptr<const ValueType>& result_type,
                  const std::vector<std::shared_ptr<op::Parameter>>& parameters,
                  const std::string& name = "");
 
         std::shared_ptr<Node> get_result() { return m_result; }
-        const std::vector<std::shared_ptr<op::Parameter>> get_parameters() const
+        const std::vector<std::shared_ptr<op::Parameter>>& get_parameters() const
         {
             return m_parameters;
         }
-        const std::shared_ptr<ValueType> get_result_type() const { return m_result_type; }
+        std::shared_ptr<const ValueType> get_result_type() const { return m_result_type; }
         std::string get_name() const;
         void set_name(const std::string& name);
-        std::list<Node*>& get_ops();
-        const std::list<Node*>& get_ops() const;
-        std::list<Node*>& get_ordered_ops();
-        const std::list<Node*>& get_ordered_ops() const;
-        void set_ordered_ops(const std::list<Node*>&);
+        std::list<std::shared_ptr<Node>>& get_ops();
+        const std::list<std::shared_ptr<Node>>& get_ops() const;
+        std::list<std::shared_ptr<Node>>& get_ordered_ops();
+        const std::list<std::shared_ptr<Node>>& get_ordered_ops() const;
+        void set_ordered_ops(const std::list<std::shared_ptr<Node>>&);
         void set_ordered_ops_valid() { m_ordered_ops_valid = true; }
         void clear_ordered_ops_valid() { m_ordered_ops_valid = false; }
         friend std::ostream& operator<<(std::ostream&, const Function&);
@@ -60,10 +60,10 @@ namespace ngraph
         std::shared_ptr<Node> m_result;
         std::vector<std::shared_ptr<ngraph::op::Parameter>> m_parameters;
         std::string m_name;
-        std::shared_ptr<ValueType> m_result_type;
+        std::shared_ptr<const ValueType> m_result_type;
         bool m_ordered_ops_valid;
-        std::list<Node*> m_ordered_ops;
-        std::list<Node*> m_ops;
+        std::list<std::shared_ptr<Node>> m_ordered_ops;
+        std::list<std::shared_ptr<Node>> m_ops;
 
     private:
         Function(const Function&) = delete;

src/ngraph/ngraph.hpp

@@ -49,12 +49,17 @@
 #include "ngraph/function.hpp"
 #include "ngraph/node.hpp"
 #include "ngraph/ops/abs.hpp"
+#include "ngraph/ops/acos.hpp"
 #include "ngraph/ops/add.hpp"
+#include "ngraph/ops/asin.hpp"
+#include "ngraph/ops/atan.hpp"
 #include "ngraph/ops/broadcast.hpp"
 #include "ngraph/ops/ceiling.hpp"
 #include "ngraph/ops/concatenate.hpp"
 #include "ngraph/ops/constant.hpp"
 #include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/cos.hpp"
+#include "ngraph/ops/cosh.hpp"
 #include "ngraph/ops/divide.hpp"
 #include "ngraph/ops/dot.hpp"
 #include "ngraph/ops/equal.hpp"
@@ -79,8 +84,14 @@
 #include "ngraph/ops/remainder.hpp"
 #include "ngraph/ops/reshape.hpp"
 #include "ngraph/ops/select.hpp"
+#include "ngraph/ops/sign.hpp"
+#include "ngraph/ops/sin.hpp"
+#include "ngraph/ops/sinh.hpp"
 #include "ngraph/ops/slice.hpp"
 #include "ngraph/ops/subtract.hpp"
+#include "ngraph/ops/sum.hpp"
+#include "ngraph/ops/tan.hpp"
+#include "ngraph/ops/tanh.hpp"
 #include "ngraph/ops/tuple.hpp"
 #include "ngraph/runtime/backend.hpp"
 #include "ngraph/runtime/call_frame.hpp"

src/ngraph/node.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
+#include "ngraph/autodiff/adjoints.hpp"
 #include "ngraph/ngraph.hpp"
 
 using namespace std;
@@ -19,7 +20,7 @@ using namespace ngraph;
 
 atomic<size_t> Node::m_next_instance_id(0);
 
-Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<ValueType> value_type)
+Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<const ValueType> value_type)
     : m_arguments(arguments)
     , m_value_type(value_type)
     , m_instance_id(m_next_instance_id.fetch_add(1))
@@ -32,6 +33,16 @@ Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<ValueType>
     }
 }
 
+Node::Node()
+    : Node({}, nullptr)
+{
+}
+
+Node::Node(std::shared_ptr<const ValueType> value_type)
+    : Node({}, value_type)
+{
+}
+
 Node::~Node()
 {
 }
@@ -51,6 +62,24 @@ void Node::set_value_type_checked(const shared_ptr<const ValueType>& value_type)
     }
 }
 
+std::shared_ptr<const ValueType> Node::get_value_type()
+{
+    if (nullptr == m_value_type)
+    {
+        propagate_types();
+    }
+    return m_value_type;
+}
+
+const std::shared_ptr<const ValueType> Node::get_value_type() const
+{
+    if (nullptr == m_value_type)
+    {
+        const_cast<Node*>(this)->propagate_types();
+    }
+    return m_value_type;
+}
+
 void Node::assign_tensors()
 {
     vector<std::shared_ptr<const TensorViewType>> tensor_view_types;
@@ -130,6 +159,18 @@ void Node::set_name(const string& name)
     }
 }
 
+std::shared_ptr<Node> Node::backprop_node(const std::shared_ptr<Node>& x,
+                                          const std::shared_ptr<Node>& c)
+{
+    auto adjoints_it = m_adjoint_map.find(c.get());
+    if (adjoints_it == m_adjoint_map.end())
+    {
+        adjoints_it =
+            m_adjoint_map.insert({c.get(), autodiff::Adjoints(shared_from_this(), c)}).first;
+    }
+    return adjoints_it->second.get(x);
+}
+
 namespace ngraph
 {
     ostream& operator<<(ostream& out, const Node& node)

src/ngraph/node.hpp

@@ -15,13 +15,17 @@
 #pragma once
 
 #include <atomic>
+#include <memory>
 #include <set>
 #include <string>
+#include <typeindex>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
 #include <iostream>
 
+#include "ngraph/autodiff/adjoints.hpp"
 #include "ngraph/common.hpp"
 #include "ngraph/descriptor/input.hpp"
 #include "ngraph/descriptor/output.hpp"
@@ -35,20 +39,21 @@ namespace ngraph
     /// view or a (possibly empty) tuple of values.
     class Node : public std::enable_shared_from_this<Node>
     {
+        // So Adjoints can call generate_adjoints
+        friend class autodiff::Adjoints;
+
     protected:
-        Node(const Nodes& arguments, std::shared_ptr<ValueType> value_type = nullptr);
-        Node()
-            : Node({}, nullptr)
-        {
-        }
+        Node(const Nodes& arguments, std::shared_ptr<const ValueType> value_type = nullptr);
+        Node();
+        Node(std::shared_ptr<const ValueType> value_type);
+
+        virtual ~Node();
 
-        Node(std::shared_ptr<ValueType> value_type)
-            : Node({}, value_type)
+        virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                       const std::shared_ptr<Node>& delta)
         {
         }
 
-        virtual ~Node();
-
     public:
         /// The class name, must not contain spaces
         virtual std::string description() const = 0;
@@ -73,11 +78,11 @@ namespace ngraph
         bool is_same_op_type(const std::shared_ptr<Node>& node) const
         {
             Node* n = node.get();
-            return typeid(*this) == typeid(*n);
+            return std::type_index(typeid(*this)) == std::type_index(typeid(*n));
         }
 
-        std::shared_ptr<const ValueType> get_value_type() { return m_value_type; }
-        const std::shared_ptr<const ValueType> get_value_type() const { return m_value_type; }
+        std::shared_ptr<const ValueType> get_value_type();
+        const std::shared_ptr<const ValueType> get_value_type() const;
         void set_value_type(const element::Type& element_type, const Shape& shape)
         {
             m_value_type = std::make_shared<TensorViewType>(element_type, shape);
@@ -109,6 +114,9 @@ namespace ngraph
         std::unordered_set<descriptor::Tensor*> liveness_new_list;
         std::unordered_set<descriptor::Tensor*> liveness_free_list;
 
+        std::shared_ptr<Node> backprop_node(const std::shared_ptr<Node>& x,
+                                            const std::shared_ptr<Node>& c);
+
     protected:
         Nodes m_arguments;
         std::shared_ptr<const ValueType> m_value_type;
@@ -119,5 +127,6 @@ namespace ngraph
         std::deque<descriptor::Input> m_inputs;
         std::deque<descriptor::Output> m_outputs;
         bool m_is_output;
+        std::unordered_map<Node*, autodiff::Adjoints> m_adjoint_map;
     };
 }

src/ngraph/ops/abs.hpp

@@ -20,9 +20,31 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise absolute value operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                      |
+        /// | ---------------------- | -------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = |\texttt{arg}[i_1,\dots,i_n]|\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                             |
+        /// | ------- | ---------------------------------- |
+        /// | NGVM    | Implemented for signed types only. |
         class Abs : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs an absolute value operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Abs(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {

src/ngraph/ops/acos.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise inverse cosine (arccos) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                             |
+        /// | ---------------------- | --------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \arccos(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Acos : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs an arccos operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Acos(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Acos"; }
+        };
+    }
+}

src/ngraph/ops/add.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/add.hpp"
+
+void ngraph::op::Add::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta);
+    adjoints.add_delta(y, delta);
+}

src/ngraph/ops/add.hpp

@@ -20,14 +20,41 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise addition operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                    |
+        /// | ---------------------- | -------------------------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n] + \texttt{arg1}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Add : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs an addition operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Add(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {
             }
             virtual std::string description() const override { return "Add"; }
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 

src/ngraph/ops/asin.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise inverse sine (arcsin) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                             |
+        /// | ---------------------- | --------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \arcsin(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Asin : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs an arcsin operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Asin(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Asin"; }
+        };
+    }
+}

src/ngraph/ops/atan.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise inverse tangent (arctan) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                             |
+        /// | ---------------------- | --------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \arctan(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Atan : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs an arctan operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Atan(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Atan"; }
+        };
+    }
+}

src/ngraph/ops/broadcast.hpp

@@ -20,15 +20,51 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Operation which "adds" axes to an input tensor, replicating elements from the input as needed along the new axes.
+        ///
+        /// Informally, a broadcast "adds" axes to the input tensor, replicating elements from the input tensor as needed to fill the new dimensions.
+        /// The parameter `m_broadcast_axes` indicates which of the output axes is being so added. For example, an output shape of `{2,5,6,2,8}` and
+        /// broadcast axes of `{1,3,4}` means that the input must have shape `{2,6}`.
+        ///
+        /// Formally, given a shape or coordinate \f$S = [d_1,\dots,d_n]\f$ and a set of axis indices \f$A\f$, define \f$\textit{del}(S,A)\f$ to be
+        /// the shape or coordinate obtained by deleting the \f$(a + 1)\f$th dimension from \f$S\f$ for each \f$a \in A\f$. Then given an input
+        /// tensor \f$T\f$ of shape \f$\textit{del}(S,A)\f$ with element type \f$E\f$, broadcasting axes \f$A\f$ produces a tensor \f$T'\f$ of shape
+        /// \f$S\f$ with element type \f$E\f$, where \f$T'[i_1,\dots,i_n] = T[del([i_1,\dots,i_n],A)]\f$.
+        ///
+        /// ## Parameters
+        ///
+        /// |                  | Description                                                              |
+        /// | ---------------- | ------------------------------------------------------------------------ |
+        /// | `shape`          | The shape \f$[d_1,\dots,d_n]\f$ of the broadcasted output.               |
+        /// | `broadcast_axes` | The indices \f$A\f$ in the `shape` of each broadcasted (i.e., new) axis. |
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                                                | Description                             |
+        /// | ----- | --------------------------------------------------- | --------------------------------------- |
+        /// | `arg` | \f$E[\mathit{del}([d_1,\dots,d_n],A)]~(n \geq 0)\f$ | A tensor of any shape and element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                     |
+        /// | ---------------------- | ------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_n]\f$ | The tensor \f$T'\f$, where \f$T'[i_1,\dots,i_n] = T[del([i_1,\dots,i_n],A)]\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                          |
+        /// | ------- | ----------------------------------------------- |
+        /// | NGVM    | Implemented for scalars, matrices, and vectors. |
+
         class Broadcast : public Builtin
         {
         public:
+            /// \brief Constructs a conversion operation.
             ///
-            /// @param arg The tensor view to be broadcast.
-            /// @param shape The shape of the result
-            /// @param broadcast_axes The axis positions (0-based) in the result that are being broadcast.
-            ///  the remaining axes in shape must be the same as the shape of arg.
-            ///
+            /// \param arg            Node that produces the input tensor to be broadcast.
+            /// \param shape          The shape of the output tensor.
+            /// \param broadcast_axes The axis positions (0-based) in the result that are being broadcast. The
+            ///                        remaining axes in shape must be the same as the shape of arg.
             Broadcast(const std::shared_ptr<Node>& arg,
                       const Shape& shape,
                       const AxisSet& broadcast_axes)
@@ -41,6 +77,7 @@ namespace ngraph
             virtual std::string description() const override { return "Broadcast"; }
             virtual void propagate_types() override;
 
+            /// \return An set containing the indices of the broadcast axes (0-based).
             const AxisSet& get_broadcast_axes() const { return m_broadcast_axes; }
         protected:
             Shape m_shape;

src/ngraph/ops/ceiling.hpp

@@ -20,9 +20,31 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise ceiling operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                  |
+        /// | ---------------------- | -------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \lceil \texttt{arg}[i_1,\dots,i_n] \rceil\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status           |
+        /// | ------- | ---------------- |
+        /// | NGVM    | Not implemented. |
         class Ceiling : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a ceiling operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Ceiling(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {

src/ngraph/ops/concatenate.hpp

@@ -20,18 +20,54 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Concatenation operation.
+        ///
+        /// Given an axis index \f$a\f$ and a rank \f$r \geq 1\f$ where \f$0 \leq a \lt r\f$, and one or more \f$r\f$-tensors
+        /// with the same element type whose shapes are the same except possibly at axis \f$a\f$, the tensors are
+        /// concatenated along axis \f$a\f$.
+        ///
+        /// For example:
+        ///   1. Concatenating matrices on axis 0 (the row axis) stacks the matrices from top to bottom.
+        ///      The number of rows in the resulting matrix is the sum of the number of rows for each
+        ///      input matrix.
+        ///   2. Concatenating matrices on axis 1 (the column axis) concatenates them from left to right.
+        ///      The number of columns in the resulting matrix is the sum of the number of columns for each
+        ///      input matrix.
+        ///   3. Concatenating 3-tensors on axis 2 (the depth axis) stacks them from front to back.
+        ///      The depth of the resulting tensor is the sum of the total depth for each input tensor.
+        ///
+        /// The resulting tensor will have the same rank as the input tensors.
+        ///
+        /// ## Parameters
+        ///
+        /// |                      | Description                                                    |
+        /// | -------------------- | -------------------------------------------------------------- |
+        /// | `concatenation_axis` | The axis \f$a\f$ along which to concatenate the input tensors. |
+        ///
+        /// ## Inputs
+        ///
+        /// |                 | Type                                                          | Description                                                                                                              |
+        /// | --------------- | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
+        /// | `args`[\f$i\f$] | \f$E[d_1,\dots,d_{a-1},d^i_a,d_{a+1},\dots,d_n]~(n \geq 1)\f$ | One or more input tensors, all of which have the same element type, and the same shape, except possibly at axis \f$a\f$. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                                         | Description                                                                                     |
+        /// | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_{a-1},\Sigma_i(d^i_a),d_{a+1},\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T\f$ is the concatenation of the input tensors along axis \f$a\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                |
+        /// | ------- | ------------------------------------- |
+        /// | NGVM    | Implemented for vectors and matrices. |
         class Concat : public Builtin
         {
         public:
-            /// Concatenates one or more tensors.
+            /// \brief Constructs a concatenation operation.
             ///
-            /// All tensors must have the same rank, and the sizes of the axes must match
-            /// everywhere except at the concatenation axis. The size of the concatenation
-            /// axis on the output is the sum of its size on all inputs; the size of other
-            /// axes is unchanged from the input tensors.
-            ///
-            /// Example: n0 has shape {2,4,2}, and n1 has shape {2,5,2}. Then the output of
-            ///          Concat(Nodes{n0,n1},1) will have shape {2,9,2}.
+            /// \param args               The nodes producing the input tensors.
+            /// \param concatenation_axis The axis along which to concatenate the input tensors.
             Concat(const Nodes& args, size_t concatenation_axis)
                 : Builtin(args)
                 , m_concatenation_axis(concatenation_axis)
@@ -41,6 +77,7 @@ namespace ngraph
             virtual std::string description() const override { return "Concatenate"; }
             virtual void propagate_types() override;
 
+            /// \return The concatenation axis.
             size_t get_concatenation_axis() const { return m_concatenation_axis; }
         protected:
             const size_t m_concatenation_axis;

src/ngraph/ops/constant.cpp

@@ -16,10 +16,35 @@
 
 using namespace ngraph::op;
 
-void ScalarConstantBase::propagate_types()
+void ConstantBase::propagate_types()
 {
 }
 
-void TensorConstantBase::propagate_types()
+template <typename ET>
+void check_value_strings(const std::vector<std::string>& value_strings)
 {
+    auto result = ET::read(value_strings);
+}
+
+void Constant::propagate_types()
+{
+    // No actual type propagation is done here; however, we check the number of value strings and
+    // also call check_value_strings just to make sure the result will be parseable at compile
+    // time. (It will throw an exception if not.)
+    auto tvt = std::dynamic_pointer_cast<const TensorViewType>(get_value_type());
+    if (nullptr == tvt)
+    {
+        throw ngraph_error("Constant does not have tensor view type");
+    }
+    auto shape = tvt->get_shape();
+
+    if (ngraph::shape_size(shape) != m_value_strings.size())
+    {
+        throw ngraph_error("Constant does not have the expected number of literals");
+    }
+
+    auto& et = tvt->get_element_type();
+
+    FUNCTION_ON_ELEMENT_TYPE(
+        et, "Constant has unhandled element type", check_value_strings, m_value_strings);
 }

src/ngraph/ops/convert.hpp

@@ -20,9 +20,41 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise type conversion operation.
+        ///
+        /// Each scalar in the input tensor is converted to the specified output element type. Note that the conversion may
+        /// result in loss of precision. For example, conversion from `float32` to `int32` is allowed.
+        ///
+        /// ## Parameters
+        ///
+        /// |                | Description                              |
+        /// | -------------- | ---------------------------------------- |
+        /// | `element_type` | The element type \f$E'\f$ to convert to. |
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                               |
+        /// | ----- | --------------------------------- | ----------------------------------------- |
+        /// | `arg` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.   |
+        ///
+        /// ## Output
+        ///
+        /// | Type                    | Description                                                                                               |
+        /// | ----------------------- | --------------------------------------------------------------------------------------------------------- |
+        /// | \f$E'[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \mathit{convert}_{(E,E')}(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Convert : public UnaryElementwiseBuiltin
         {
         public:
+            /// \brief Constructs a conversion operation.
+            ///
+            /// \param arg          Node that produces the input tensor.
+            /// \param element_type Element type for the output tensor.
             Convert(const std::shared_ptr<Node>& arg, const ngraph::element::Type& element_type)
                 : UnaryElementwiseBuiltin({arg})
                 , m_element_type(element_type)

src/ngraph/ops/cos.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise cosine operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                          |
+        /// | ---------------------- | ------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \cos(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Cos : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a cosine operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Cos(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Cos"; }
+        };
+    }
+}

src/ngraph/ops/cosh.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise hyperbolic cosine (cosh) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                           |
+        /// | ---------------------- | ------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \cosh(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Cosh : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a hyperbolic cosine operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Cosh(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Cosh"; }
+        };
+    }
+}

src/ngraph/ops/divide.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Divide::generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta * shared_from_this() / x);
+    adjoints.add_delta(y, -delta * shared_from_this() / y);
+}

src/ngraph/ops/divide.hpp

@@ -20,14 +20,41 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise division operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                              |
+        /// | ---------------------- | ------------------------------------------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n] \mathbin{/} \texttt{arg1}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Divide : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a division operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Divide(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Divide"; }
         };
     }

src/ngraph/ops/dot.hpp

@@ -20,28 +20,95 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Inner product/dot product/matrix product/tensor contraction operation.
+        ///
+        /// Takes two arguments `arg0` and `arg1`. There are three possible cases:
+        ///
+        /// 1. `arg0` or `arg1` is 0-dimensional. Then, treats that 0-dimensional argument as a scalars and computes a scalar-tensor product.
+        ///     (Example: `arg0` has shape `{1,2,3}` and arg1 has shape `{}`; then the result will have shape `{1,2,3}`.)
+        ///
+        /// 2. `arg1` is a vector (1-dimensional tensor). Then, computes a dot product reducing on the innermost (rightmost) dimensions of `arg0` and `arg1`.
+        ///         (Example: arg0 has shape `{1,2,3}` and arg1 has shape `{3}`; then the result will have shape `{1,2}`.)
+        ///
+        /// 3. `arg1` is more than 1-dimensional. Then, computes a dot product reducing on the innermost (rightmost) dimension of arg0, and the next-to-innermost dimension of arg1.
+        ///         (Example: arg0 has shape {3,4} and arg1 has shape {4,3}; then the result will have shape {3,3}.)
+        ///
+        ///
+        /// = Case 1: Scalar-tensor product =
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                                  |
+        /// | ------ | --------------------------------- | ------------------------------------------------------------ |
+        /// | `arg0` | \f$E[]\f$                         | A scalar of any element type.                                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape, with the same element type as `arg0`. |
+        ///
+        /// <i>(Note: the order of inputs may be reversed in this case, i.e., `arg1` can be the scalar and `arg0` the tensor.)</i>
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                          |
+        /// | ---------------------- | ---------------------------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \mathtt{arg0} \cdot \mathtt{arg1}[i_1,\dots,i_n]\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        ///
+        /// = Case 2: Vector-tensor product =
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                | Description                                                                                                  |
+        /// | ------ | ----------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+        /// | `arg0` | \f$E[d]\f$                          | A vector of any element type.                                                                                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n,d]~(n \geq 0)\f$ | A tensor of any shape whose innermost dimension matches `arg0`'s size, with the same element type as `arg0`. |
+        ///
+        /// <i>(Note: in the particular case where \f$n = 0\f$, this is a vector dot product; when \f$n = 1\f$, this is a vector-matrix product.)</i>
+        //
+        /// ## Output
+        ///
+        /// | Type                                                          | Description                                                   |
+        /// | ---------------------- | ---------------------------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \mathtt{arg0} \cdot \mathtt{arg1}[i_1,\dots,i_n]\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                      |
+        /// | ------- | ------------------------------------------- |
+        /// | NGVM    | Implemented for `arg1` with rank 2 or less. |
+        ///
+        /// = Case 3: Tensor-tensor product =
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                                        | Description                                                                                                                                                  |
+        /// | ------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 1)\f$                           | A tensor of any shape with rank of at least 1, and any element type.                                                                                         |
+        /// | `arg1` | \f$E[d'_1,\dots,d'_m]~(m \geq 2\text{ and }d'_{m-1}=d_n)\f$ | A tensor with the same element type as `arg0`, and any shape with rank of at least 2 whose next-to-innermost dimension matches `arg0`'s innermost dimension. |
+        ///
+        /// <i>(Note: in the particular case where \f$n = m = 2\f$, this is a matrix product.)</i>
+        ///
+        /// ## Output
+        ///
+        /// | Type                                                  | Description                                                                                                                       |
+        /// | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------|
+        /// | \f$E[d_1,\dots,d_{n-1},d'_1,\dots,d'_{m-2},d'_{m}]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_{n-1},j_1,\dots,j_{m-2},j_m] = \dots\f$ TODO: FIXME: finish this; but it's like numpy. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                         |
+        /// | ------- | ---------------------------------------------- |
+        /// | NGVM    | Implemented for `arg1` with rank of exactly 2. |
         class Dot : public Builtin
         {
         public:
-            /// Computes the dot product of two tensors.
+            /// \brief Constructs a dot product operation.
             ///
-            /// There are three possible cases:
-            ///  (1) arg0 or arg1 is 0-dimensional. Then, we treat the 0-dimensional
-            ///      argument(s) as scalars and compute a scalar-tensor or
-            ///      scalar-scalar product.
-            ///         (Example: arg0 has shape {1,2,3} and arg1 has shape {}; then
-            ///         the result will have shape {1,2,3}.)
-            ///
-            ///  (2) arg1 is 1-dimensional. Then, we compute a dot product reducing
-            ///      on the innermost (rightmost) dimensions of arg0 and arg1.
-            ///         (Example: arg0 has shape {1,2,3} and arg1 has shape {3}; then
-            ///         the result will have shape {1,2}.)
-            ///
-            ///  (3) arg1 is more than 1-dimensional. Then, we compute a dot product
-            ///      reducing on the innermost (rightmost) dimension of arg0, and the
-            ///      next-to-innermost dimension of arg1.
-            ///         (Example: arg0 has shape {3,4} and arg1 has shape {4,3}; then
-            ///         the result will have shape {3,3}.)
+            /// \param arg0 The node producing the first argument.
+            /// \param arg1 The node producing the second argument.
             Dot(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : Builtin({arg0, arg1})
             {

src/ngraph/ops/equal.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise is-equal operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                |
+        /// | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] = \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Equal : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs an is-equal operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Equal(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/exp.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/exp.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Exp::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * shared_from_this());
+}

src/ngraph/ops/exp.hpp

@@ -20,14 +20,39 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise natural exponential (exp) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                          |
+        /// | ---------------------- | ------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \exp(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Exp : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs an exponential operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Exp(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Exp"; }
         };
     }

src/ngraph/ops/floor.hpp

@@ -20,9 +20,31 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise floor operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                    |
+        /// | ---------------------- | ---------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \lfloor \texttt{arg}[i_1,\dots,i_n] \rfloor\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status           |
+        /// | ------- | ---------------- |
+        /// | NGVM    | Not implemented. |
         class Floor : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a floor operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Floor(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {

src/ngraph/ops/function_call.hpp

@@ -21,14 +21,39 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief %Function call operation.
+        ///
+        /// ## Parameters
+        ///
+        /// |            | Description                |
+        /// | ---------- | -------------------------- |
+        /// | `function` | The function to be called. |
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                                                                                                                                                       | Description                          |
+        /// | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ |
+        /// | `args` | \f$T_1,\dots,T_n\f$ where \f$n\f$ matches the number of arguments expected by `function` and \f$T_i\f$ matches the type expected for the \f$i\f$th argument of `function`. | The arguments for the function call. |
+        ///
+        /// ## Output
+        ///
+        /// | Type      | Description                                              |
+        /// | --------- | -------------------------------------------------------- |
+        /// | \f$T_R\f$ | The tensor returned by `function` when called on `args`. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class FunctionCall : public Builtin
         {
         public:
+            /// \brief Constructs a function call operation.
             ///
-            /// @param function The function to be called
-            /// @param args The function arguments
-            ///
-            FunctionCall(const std::shared_ptr<Function>& function,
+            /// \param function The function to be called.
+            /// \param args The arguments for the function call.
+            FunctionCall(std::shared_ptr<Function> function,
                          const std::vector<std::shared_ptr<Node>>& args)
                 : Builtin(args)
                 , m_function(function)
@@ -38,6 +63,7 @@ namespace ngraph
             virtual std::string description() const override { return "FunctionCall"; }
             virtual void propagate_types() override;
 
+            /// \return The function to be called.
             std::shared_ptr<Function> get_function() const { return m_function; }
         protected:
             std::shared_ptr<Function> m_function;

src/ngraph/ops/get_tuple_element.hpp

@@ -22,9 +22,38 @@ namespace ngraph
     {
         class Node;
 
+        /// \brief Operation to get an element from a tuple.
+        ///
+        /// ## Parameters
+        ///
+        /// |     | Description                                                        |
+        /// | --- | ------------------------------------------------------------------ |
+        /// | `n` | The position of the element (0-based) to get from the input tuple. |
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                                        | Description                                |
+        /// | ------ | ----------------------------------------------------------- | ------------------------------------------ |
+        /// | `arg`  | \f$(T_1,\dots,T_{n-1},T_n,T_{n+1},\dots,T_m)~(m \geq 1)\f$ | An input tuple with at least `n` elements. |
+        ///
+        /// ## Output
+        ///
+        /// | Type      | Description                           |
+        /// | --------- | ------------------------------------- |
+        /// | \f$T_n\f$ | The `n`th element of the input tuple. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class GetTupleElement : public Builtin
         {
         public:
+            /// \brief Constructs a get-tuple-element operation.
+            ///
+            /// \param arg The input tuple.
+            /// \param n The index of the tuple element to get.
             GetTupleElement(const std::shared_ptr<Node>& arg, size_t n)
                 : Builtin({arg})
                 , m_n{n}
@@ -33,6 +62,7 @@ namespace ngraph
 
             virtual void propagate_types() override;
             virtual std::string description() const override { return "GetTupleElement"; }
+            /// \return The index of the tuple element to get.
             size_t get_n() const { return m_n; }
         protected:
             size_t m_n;

src/ngraph/ops/greater.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise greater-than operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                  |
+        /// | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] \gt \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Greater : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs a greater-than operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Greater(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/greater_eq.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise greater-than-or-equal operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                   |
+        /// | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] \geq \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class GreaterEq : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs a greater-than-or-equal operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             GreaterEq(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/less.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise less-than operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                  |
+        /// | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] \lt \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Less : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs a less-than operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Less(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/less_eq.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise less-than-or-equal operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                   |
+        /// | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] \leq \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class LessEq : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs a less-than-or-equal operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             LessEq(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/log.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/log.hpp"
+#include "ngraph/ops/divide.hpp"
+
+void ngraph::op::Log::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta / x);
+}

src/ngraph/ops/log.hpp

@@ -20,14 +20,39 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise natural log operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                         |
+        /// | ---------------------- | ----------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \ln(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Log : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a natural log operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Log(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Log"; }
         };
     }

src/ngraph/ops/maximum.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise maximum operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                        |
+        /// | ---------------------- | ------------------------------------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \max(\texttt{arg0}[i_1,\dots,i_n],\texttt{arg1}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Maximum : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a maximum operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Maximum(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {

src/ngraph/ops/minimum.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise minimum operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                        |
+        /// | ---------------------- | ------------------------------------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \min(\texttt{arg0}[i_1,\dots,i_n],\texttt{arg1}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Minimum : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a minimum operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Minimum(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {

src/ngraph/ops/multiply.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Multiply::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta * y);
+    adjoints.add_delta(y, x * delta);
+}

src/ngraph/ops/multiply.hpp

@@ -20,17 +20,44 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise multiplication operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                        |
+        /// | ---------------------- | ------------------------------------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n] \cdot \texttt{arg1}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Multiply : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a multiplication operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Multiply(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {
             }
 
             virtual std::string description() const override { return "Multiply"; }
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
-    }
+    };
 
     inline std::shared_ptr<ngraph::Node> operator*(const std::shared_ptr<ngraph::Node> arg0,
                                                    const std::shared_ptr<ngraph::Node> arg1)

src/ngraph/ops/negative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Negative::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, -delta);
+}

src/ngraph/ops/negative.hpp

@@ -20,14 +20,39 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise negation operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                       |
+        /// | ---------------------- | --------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = -(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Negative : public UnaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a negation operation.
+            ///
+            /// \param arg Node that produces the input tensor.
             Negative(const std::shared_ptr<Node>& arg)
                 : UnaryElementwiseArithmetic(arg)
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Negative"; }
         };
     }

src/ngraph/ops/not_equal.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise not-equal operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and element type.                |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                                                   |
+        /// | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg0}[i_1,\dots,i_n] \neq \texttt{arg1}[i_1,\dots,i_n]\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class NotEqual : public BinaryElementwiseComparison
         {
         public:
+            /// \brief Constructs a not-equal operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             NotEqual(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseComparison(arg0, arg1)
             {

src/ngraph/ops/parameter.cpp

@@ -19,10 +19,8 @@
 using namespace std;
 using namespace ngraph::op;
 
-Parameter::Parameter(const std::shared_ptr<ValueType>& value_type)
+Parameter::Parameter(const std::shared_ptr<const ValueType>& value_type)
     : Node(value_type)
-    , m_function(nullptr)
-    , m_index(0)
 {
 }
 
@@ -31,16 +29,10 @@ Parameter::Parameter(const ngraph::element::Type& element_type, const Shape& sha
 {
 }
 
-void Parameter::assign_function(Function* function, size_t index)
+void Parameter::propagate_types()
 {
-    if (nullptr != m_function)
-    {
-        throw ngraph_error("Re-assigning function to a parameter.");
-    }
-    m_function = function;
-    m_index = index;
 }
 
-void Parameter::propagate_types()
+void Parameter::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
 {
 }

src/ngraph/ops/parameter.hpp

@@ -22,30 +22,48 @@ namespace ngraph
     class Function;
     namespace op
     {
+        /// \brief A function parameter.
         ///
         /// Parameters are nodes that represent the arguments that will be passed to user-defined functions.
         /// Function creation requires a sequence of parameters.
         /// Basic graph operations do not need parameters attached to a function.
         ///
+        /// ## Parameters
+        ///
+        /// |              | Description                        |
+        /// | -------------| ---------------------------------- |
+        /// | `value_type` | The type \f$T\f$ of the parameter. |
+        ///
+        /// ## Output
+        ///
+        /// | Type    | Description                                                                                                                 |
+        /// | ------- | --------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$T\f$ | The value of the parameter, supplied by the `FunctionCall` to this function or in the initial `ngraph::runtime::CallFrame`. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Parameter : public Node
         {
-            friend class ngraph::Function;
-
         protected:
-            // Called by the Function constructor to associate this parameter with the function.
-            // It is an error to try to associate a parameter with more than one function.
-            void assign_function(Function* function, size_t index);
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
 
         public:
-            Parameter(const std::shared_ptr<ValueType>& value_type = nullptr);
+            /// \brief Constructions a parameter node.
+            ///
+            /// \param value_type The type of the parameter.
+            Parameter(const std::shared_ptr<const ValueType>& value_type = nullptr);
+            /// \brief Constructions a tensor view-typed parameter node.
+            ///
+            /// \param element_type The element type of the parameter.
+            /// \param shape The shape of the parameter.
             Parameter(const ngraph::element::Type& element_type, const Shape& shape);
 
             std::string description() const override { return "Parameter"; }
             virtual void propagate_types() override;
-
-        protected:
-            Function* m_function;
-            size_t m_index;
         };
     }
 }

src/ngraph/ops/power.hpp

@@ -20,9 +20,33 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise exponentiation operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                    |
+        /// | ---------------------- | -------------------------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n]^{\texttt{arg1}[i_1,\dots,i_n]}\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status           |
+        /// | ------- | ---------------- |
+        /// | NGVM    | Not implemented. |
         class Power : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs an exponentiation operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Power(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {

src/ngraph/ops/reduce.hpp

@@ -20,15 +20,82 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Tensor reduction operation.
+        ///
+        /// Element-wise reduces the input tensor, eliminating the specified reduction axes, given a reduction function that maps two scalars to a scalar.
+        /// For example, if the reduction function \f$f(x,y) = x+y\f$:
+        ///
+        /// \f[
+        ///     \mathit{reduce}\left(f,\{0\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///     \left[ (1 + 3 + 5), (2 + 4 + 6) \right] =
+        ///     \left[ 9, 12 \right]~~~\text{(dimension 0 (rows) is eliminated)}
+        /// \f]
+        ///
+        /// \f[
+        ///     \mathit{reduce}\left(f,\{1\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///     \left[ (1 + 2), (3 + 4), (5 + 6) \right] =
+        ///     \left[ 3, 7, 11 \right]~~~\text{(dimension 1 (columns) is eliminated)}
+        /// \f]
+        ///
+        /// \f[
+        ///     \mathit{reduce}\left(f,\{0,1\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///      (1 + 2) + (3 + 4) + (5 + 6) =
+        ///      21~~~\text{(both dimensions (rows and columns) are eliminated)}
+        /// \f]
+        ///
+        /// It is assumed that \f$f\f$ is associative. In other words, the order of operations is undefined. In the case where a collapsed dimension is 0,
+        /// the value of `arg_init` will be substituted.
+        ///
+        /// Note that the parameter `reduction_axes` specifies which axes are to be <i>eliminated</i>, which can be a bit counterintuitive. For example,
+        /// as seen above, eliminating the column dimension results in the <i>rows</i> being summed, not the columns.
+        ///
+        /// ## Parameters
+        ///
+        /// |                      | Description                                                                                                               |
+        /// | -------------------- | ------------------------------------------------------------------------------------------------------------------------- |
+        /// | `reduction_function` | The scalar function used to reduce the input tensor. Must take two arguments of type \f$E[]\f$ and return type \f$E[]\f$. |
+        /// | `reduction_axes`     | The axes to eliminate through reduction.                                                                                  |
+        ///
+        /// ## Inputs
+        ///
+        /// |                | Type                              | Description                                                                                           |
+        /// | -------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
+        /// | `arg_reductee` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape, with the element type matching that expected by the reduction function. |
+        /// | `arg_init`     | \f$E[]\f$                         | An scalar to be used as a substitute output value on zero-sized axes.                                 |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by reduction. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                                |
+        /// | ------- | ----------------------------------------------------- |
+        /// | NGVM    | Fully implemented for scalars, vectors, and matrices. |
+
         class Reduce : public Builtin
         {
         public:
+            /// \brief Constructs a reduction operation.
             ///
-            /// @param arg_reductee The tensor view to be reduced.
-            /// @param arg_init The initial value for reduction.
-            /// @param reduction_function The reduction function to use.
-            /// @param reduction_axes The axis positions (0-based) to be reduced.
-            ///
+            /// \param arg_reductee The tensor view to be reduced.
+            /// \param arg_init The initial value for reduction.
+            /// \param reduction_function The reduction function to use.
+            /// \param reduction_axes The axis positions (0-based) to be eliminated.
             Reduce(const std::shared_ptr<Node>& arg_reductee,
                    const std::shared_ptr<Node>& arg_init,
                    const std::shared_ptr<Function>& reduction_function,
@@ -42,10 +109,12 @@ namespace ngraph
             virtual std::string description() const override { return "Reduce"; }
             virtual void propagate_types() override;
 
+            /// \return The function to use for reduction.
             std::shared_ptr<Function> get_reduction_function() const
             {
                 return m_reduction_function;
             }
+            /// \return The axis positions (0-based) to be eliminated through reduction.
             const AxisSet& get_reduction_axes() const { return m_reduction_axes; }
         protected:
             std::shared_ptr<Function> m_reduction_function;

src/ngraph/ops/remainder.hpp

@@ -20,9 +20,35 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise remainder operation.
+        ///
+        /// (TODO: Get a bit more clarity on this: is it just "mod"? What about negative numbers and floats?)
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                       |
+        /// | ---------------------- | ----------------------------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n] \mod \texttt{arg1}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status           |
+        /// | ------- | ---------------- |
+        /// | NGVM    | Not implemented. |
         class Remainder : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs a remainder operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Remainder(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {

src/ngraph/ops/reshape.hpp

@@ -20,16 +20,55 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Tensor reshape operation.
+        ///
+        /// "Converts" an input tensor into a new shape with the same number of elements.
+        ///
+        /// Given that the input tensor has shape \f$[d_1,\dots,d_n]\f$, the output may have any shape \f$[d'_1,\dots,d'_m]\f$ such that
+        /// \f$\Pi_{0 \leq i \lt n}(d_i) = \Pi_{0 \leq i \lt m}(d'_i)\f$. For example, a \f$3\times{}4\f$ matrix can be reshaped into a
+        /// 3-tensor of shape \f$3\times{}2\times{}2\f$, a matrix of shape \f$6\times{}2\f$, or a vector of size \f$12\f$, but not, for
+        /// example, a matrix of size \f$4\times{}4\f$.
+        ///
+        /// The parameter `input_order` indicates the order in which to "walk" over the input axes. Given a tensor of shape \f$(d_1,\dots,d_n)\f$,
+        /// an input order of \f$(a_0, a_1, \dots, a_{n-1})\f$ results in the coordinate for axis \f$a_{n-1}\f$ being varied most frequently,
+        /// followed by axis \f$a-2\f$, and so on down to \f$a_0\f$.
+        ///
+        /// (TODO: example.)
+        ///
+        /// ## Parameters
+        ///
+        /// |                | Description                                                |
+        /// | -------------- | ---------------------------------------------------------- |
+        /// | `input_order`  | The order in which to walk over the input axes.            |
+        /// | `output_shape` | The shape \f$[d'_1,\dots,d'_m]\f$ for the reshaped tensor. |
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                                                                                  |
+        /// | ----- | --------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+        /// | `arg` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any type and shape, as long as the product of \f$d_i\f$ equals the product of \f$d'_i\f$. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                     | Description                                                                                            |
+        /// | ------------------------ | ------------------------------------------------------------------------------------------------------ |
+        /// | \f$E[d'_1,\dots,d'_m]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with its elements rearranged as described above. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                                                                                                                                                            |
+        /// | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | NGVM    | Fully implemented for scalars, vectors, and matrices. Implemented for other shapes only when there is no reordering of the input axes, i.e. `input_order` is \f$(0,\dots,n-1)\f$. |
         class Reshape : public Builtin
         {
         public:
+            /// \brief Constructs a reshape operation.
             ///
-            /// @param arg The tensor view to be reshaped.
-            /// @param input_order The order in which to iterate over input axes. (TODO: that needs more explanation)
-            ///        This must be a permutation of the sequence (0,...,n-1) where n is the rank of the input tensor.
-            /// @param output_shape The output shape. If the input shape is (a0,...,ak-1) then the output shape must
-            ///        be of the form (b0,...,bj-1) where product(ai) == product(bi).
-            ///
+            /// \param arg The tensor view to be reshaped.
+            /// \param input_order The order in which to iterate over input axes. This must be a permutation of the
+            ///                    sequence \f$(0,\dots,n-1)\f$ where \f$n\f$ is the rank of the input tensor.
+            /// \param output_shape The output shape. If the input shape is \f$(a_0,\dots,a_{k-1})\f$ then the output shape must
+            ///        be of the form \f$(b_0,\dots,b_{j-1})\f$ where \f$\Pi(a_i) = \Pi(b_i)\f$.
             Reshape(const std::shared_ptr<Node>& arg,
                     const AxisVector& input_order,
                     const Shape& output_shape)
@@ -42,7 +81,9 @@ namespace ngraph
             virtual std::string description() const override { return "Reshape"; }
             virtual void propagate_types() override;
 
+            /// \return The order in which to iterate over input axes.
             const AxisVector& get_input_order() const { return m_input_order; }
+            /// \return The shape of the output tensor.
             const Shape& get_output_shape() const { return m_output_shape; }
         protected:
             const AxisVector m_input_order;

src/ngraph/ops/select.hpp

@@ -20,9 +20,35 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise selection operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                          | Description                                                  |
+        /// | ------ | --------------------------------------------- | ------------------------------------------------------------ |
+        /// | `arg0` | \f$\texttt{bool}[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape, with element `bool`.                  |
+        /// | `arg1` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$             | A tensor of the same shape as `arg0`, with any element type. |
+        /// | `arg2` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$             | A tensor of the same shape and element type as `arg1`.       |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                                                                             |
+        /// | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg1}[i_1,\dots,i_n]\text{ if }\texttt{arg0}[i_1,\dots,i_n] \neq 0\text{, else }\texttt{arg2}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Select : public Builtin
         {
         public:
+            /// \brief Constructs a selection operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
+            /// \param arg2 Node that produces the third input tensor.
             Select(const std::shared_ptr<Node>& arg0,
                    const std::shared_ptr<Node>& arg1,
                    const std::shared_ptr<Node>& arg2)

src/ngraph/ops/sign.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise sign operation.
+        ///
+        /// Maps each element of the input tensor to -1 (if it is negative), 0 (if it is zero), or 1 (if it is positive).
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                          |
+        /// | ---------------------- | ------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \text{sgn}(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Sign : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs an elementwise sign operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Sign(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Sign"; }
+        };
+    }
+}

src/ngraph/ops/sin.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise sine operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                          |
+        /// | ---------------------- | ------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \sin(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Sin : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a sine operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Sin(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Sin"; }
+        };
+    }
+}

src/ngraph/ops/sinh.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise hyperbolic sine (sinh) operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                           |
+        /// | ---------------------- | ------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \sinh(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Sinh : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a hyperbolic sine operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Sinh(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Sinh"; }
+        };
+    }
+}

src/ngraph/ops/slice.hpp

@@ -20,16 +20,48 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Takes a slice of an input tensor, i.e., the sub-tensor that resides within a bounding box, optionally with stride.
+        ///
+        /// Given an input tensor \f$T\f$ of shape \f$[d_1,\dots,d_n]\f$, lower bounds \f$[l_1,\dots,l_n]\f$, and upper bounds \f$[u_1,\dots,u_n]\f$,
+        /// where \f$l_i \leq d_i \leq d_i\f$, and a stride \f$[s_1,\dots,s_n]\f$, returns a new tensor \f$T'\f$ of the same element type and shape
+        /// \f$[d'_1,\dots,d'_n]\f$ where \f$d'_i = \lceil(u_i - l_i)\, /\, s_i\rceil\f$, where \f$T'[i_1,\dots,i_n] = T[i'_1,\dots,i'_n]\f$
+        /// where \f$i'_j = i_j s_j + l_j\f$.
+        ///
+        /// ## Parameters
+        ///
+        /// |                | Description                                                                                                                                                                |
+        /// | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | `lower_bounds` | The (inclusive) lower-bound coordinates \f$l_i\f$ for the tensor slice. For example, a lower-bound of \f$(1,2)\f$ means to start the slice at row 1 and column 2.          |
+        /// | `upper_bounds` | The (non-inclusive) upper-bound coordinates \f$u_i\f$ for the tensor slice. For example, an upper-bound of \f$(5,4)\f$ means to end the slice before row 4 and column 3.   |
+        /// | `step`         | The "step" or "stride" \f$s_i\f$ for the tensor slice. For example, a stride of \f$(1,3)\f$ means to take every row, and every third column (starting at the lower bound). |
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                                                | Description                             |
+        /// | ----- | --------------------------------------------------- | --------------------------------------- |
+        /// | `arg` | \f$E[\mathit{del}([d_1,\dots,d_n],A)]~(n \geq 0)\f$ | A tensor of any shape and element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                                                           | Description                       |
+        /// | ------------------------------------------------------------------------------ | --------------------------------- |
+        /// | \f$E[d'_1,\dots,d'_n]\f$ where \f$d'_i = \lceil(u_i - l_i)\, /\, s_i\rceil\f$. | The tensor sliced from the input. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                          |
+        /// | ------- | ----------------------------------------------- |
+        /// | NGVM    | Implemented for scalars, matrices, and vectors. |
         class Slice : public Builtin
         {
         public:
+            /// \brief Constructs a tensor slice operation.
             ///
-            /// @param arg The tensor view to be sliced.
-            /// @param lower_bounds The axiswise lower bounds of the slice.
-            /// @param upper_bounds The axiswise upper bounds of the slice (exclusive).
-            /// @param step The slicing step; for example, step of {n,m} means to take
-            ///             every nth row and everyth mth column of the input matrix.
-            ///
+            /// \param arg The tensor view to be sliced.
+            /// \param lower_bounds The axiswise lower bounds of the slice (inclusive).
+            /// \param upper_bounds The axiswise upper bounds of the slice (exclusive).
+            /// \param step The slicing step; for example, step of `{n,m}` means to take
+            ///             every nth row and every mth column of the input matrix.
             Slice(const std::shared_ptr<Node>& arg,
                   const Coordinate& lower_bounds,
                   const Coordinate& upper_bounds,
@@ -41,6 +73,11 @@ namespace ngraph
             {
             }
 
+            /// \brief Constructs a tensor slice operation with unit step; i.e., every element inside the bounding box will be copied to the output slice.
+            ///
+            /// \param arg The tensor view to be sliced.
+            /// \param lower_bounds The axiswise lower bounds of the slice (inclusive).
+            /// \param upper_bounds The axiswise upper bounds of the slice (exclusive).
             Slice(const std::shared_ptr<Node>& arg,
                   const Coordinate& lower_bounds,
                   const Coordinate& upper_bounds)
@@ -54,8 +91,11 @@ namespace ngraph
             virtual std::string description() const override { return "Slice"; }
             virtual void propagate_types() override;
 
+            /// \return The inclusive lower-bound coordinates.
             const Coordinate& get_lower_bounds() const { return m_lower_bounds; }
+            /// \return The exclusive upper-bound coordinates.
             const Coordinate& get_upper_bounds() const { return m_upper_bounds; }
+            /// \return The slicing step.
             const Shape& get_step() const { return m_step; }
         protected:
             const Coordinate m_lower_bounds;

src/ngraph/ops/subtract.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/subtract.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Subtract::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta);
+    adjoints.add_delta(y, -delta);
+}

src/ngraph/ops/subtract.hpp

@@ -20,14 +20,41 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Elementwise subtraction operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                              | Description                                            |
+        /// | ------ | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg0` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type.        |
+        /// | `arg1` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of the same shape and element type as `arg0`. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                    |
+        /// | ---------------------- | -------------------------------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \texttt{arg0}[i_1,\dots,i_n] - \texttt{arg1}[i_1,\dots,i_n]\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Subtract : public BinaryElementwiseArithmetic
         {
         public:
+            /// \brief Constructs an subtraction operation.
+            ///
+            /// \param arg0 Node that produces the first input tensor.
+            /// \param arg1 Node that produces the second input tensor.
             Subtract(const std::shared_ptr<Node>& arg0, const std::shared_ptr<Node>& arg1)
                 : BinaryElementwiseArithmetic(arg0, arg1)
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Subtract"; }
         };
     }

src/ngraph/ops/sum.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/sum.hpp"
+#include "ngraph/function.hpp"
+
+using namespace std;
+using namespace ngraph::op;
+
+void Sum::propagate_types()
+{
+    if (m_arguments.size() != 1)
+    {
+        throw ngraph_error("Wrong number of arguments.");
+    }
+
+    auto arg_type = m_arguments.at(0)->get_value_type();
+    if (nullptr == arg_type)
+    {
+        throw ngraph_error("Argument to sum is missing type.");
+    }
+    auto arg_tensor_view_type = dynamic_pointer_cast<const TensorViewType>(arg_type);
+    if (nullptr == arg_tensor_view_type)
+    {
+        throw ngraph_error("Argument to sum is not a tensor view");
+    }
+
+    auto& arg_element_type = arg_tensor_view_type->get_element_type();
+    if (arg_element_type == element::Bool::element_type())
+    {
+        throw ngraph_error("Argument for sum must have numeric element type");
+    }
+
+    auto arg_shape = arg_tensor_view_type->get_shape();
+
+    for (auto axis : m_reduction_axes)
+    {
+        if (axis >= arg_shape.size())
+        {
+            throw ngraph_error("Reduction axis for sum is out of bounds");
+        }
+    }
+
+    Shape result_shape;
+
+    for (size_t i = 0; i < arg_shape.size(); i++)
+    {
+        if (m_reduction_axes.count(i) == 0)
+        {
+            result_shape.push_back(arg_shape.at(i));
+        }
+    }
+
+    set_value_type_checked(
+        make_shared<TensorViewType>(arg_tensor_view_type->get_element_type(), result_shape));
+}

src/ngraph/ops/sum.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Tensor sum operation.
+        ///
+        /// Element-wise sums the input tensor, eliminating the specified reduction axes.
+        /// For example:
+        ///
+        /// \f[
+        ///     \mathit{sum}\left(\{0\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///     \left[ (1 + 3 + 5), (2 + 4 + 6) \right] =
+        ///     \left[ 9, 12 \right]~~~\text{(dimension 0 (rows) is eliminated)}
+        /// \f]
+        ///
+        /// \f[
+        ///     \mathit{sum}\left(\{1\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///     \left[ (1 + 2), (3 + 4), (5 + 6) \right] =
+        ///     \left[ 3, 7, 11 \right]~~~\text{(dimension 1 (columns) is eliminated)}
+        /// \f]
+        ///
+        /// \f[
+        ///     \mathit{sum}\left(\{0,1\},
+        ///         \left[ \begin{array}{ccc}
+        ///                1 & 2 \\
+        ///                3 & 4 \\
+        ///                5 & 6 \end{array} \right]\right) =
+        ///      (1 + 2) + (3 + 4) + (5 + 6) =
+        ///      21~~~\text{(both dimensions (rows and columns) are eliminated)}
+        /// \f]
+        ///
+        /// This is equivalent to Reduce where `arg_init` = 0 and `reduction_function` is \f$f(x,y) = x+y\f$.
+        ///
+        /// ## Parameters
+        ///
+        /// |                      | Description                              |
+        /// | -------------------- | ---------------------------------------- |
+        /// | `reduction_axes`     | The axes to eliminate through summation. |
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                            |
+        /// | ----- | --------------------------------- | ------------------------------------------------------ |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$N[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by summation. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                                |
+        /// | ------- | ----------------------------------------------------- |
+        /// | NGVM    | Fully implemented for scalars, vectors, and matrices. |
+        class Sum : public Builtin
+        {
+        public:
+            /// \brief Constructs a summation operation.
+            ///
+            /// \param arg The tensor view to be summed.
+            /// \param reduction_axes The axis positions (0-based) to be eliminated.
+            Sum(const std::shared_ptr<Node>& arg, const AxisSet& reduction_axes)
+                : Builtin({arg})
+                , m_reduction_axes(reduction_axes)
+            {
+            }
+
+            virtual std::string description() const override { return "Sum"; }
+            virtual void propagate_types() override;
+
+            /// \return The axis positions (0-based) to be eliminated through summation.
+            const AxisSet& get_reduction_axes() const { return m_reduction_axes; }
+        protected:
+            AxisSet m_reduction_axes;
+        };
+    }
+}

src/ngraph/ops/tan.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise tangent operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                          |
+        /// | ---------------------- | ------------------------------------------------------------------------------------ |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \tan(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Tan : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a tangent operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Tan(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Tan"; }
+        };
+    }
+}

src/ngraph/ops/tanh.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise hyperbolic tangent operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                           |
+        /// | ---------------------- | ------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \tanh(\texttt{arg}[i_1,\dots,i_n])\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Tanh : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a hyperbolic tangent operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Tanh(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic(arg)
+            {
+            }
+
+            virtual std::string description() const override { return "Tanh"; }
+        };
+    }
+}

src/ngraph/ops/tuple.hpp

@@ -20,9 +20,31 @@ namespace ngraph
 {
     namespace op
     {
+        /// \brief Operation to construct a tuple.
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                           | Description                            |
+        /// | ------ | ------------------------------ | -------------------------------------- |
+        /// | `args` | \f$T_1,\dots,T_n~(n \geq 0)\f$ | The elements of the constructed tuple. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                  | Description                                                |
+        /// | --------------------- | ---------------------------------------------------------- |
+        /// | \f$(T_1,\dots,T_n)\f$ | The tuple \f$(\texttt{args}[0],\dots,\texttt{args}[n])\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Tuple : public Builtin
         {
         public:
+            /// \brief Constructs a tuple construction operation.
+            ///
+            /// \param args The nodes that produce the elements of the constructed tuple.
             Tuple(const Nodes& args)
                 : Builtin(args)
             {

src/ngraph/pass/assign_tensors.cpp

@@ -25,15 +25,15 @@
 using namespace std;
 using namespace ngraph;
 
-bool pass::AssignTensors::run_on_call_graph(list<Node*>& nodes)
+bool pass::AssignTensors::run_on_call_graph(list<std::shared_ptr<Node>>& nodes)
 {
-    for (Node* node : nodes)
+    for (shared_ptr<Node> node : nodes)
     {
         try
         {
             // We need to set the nodes is_output state prior to call assign_tensors
             // so that the output state can be passes to the constructed tensors.
-            if (node == get_state().get_functions().at(0)->get_result().get())
+            if (node == get_state().get_functions().at(0)->get_result())
             {
                 node->set_is_output();
             }

src/ngraph/pass/assign_tensors.hpp

@@ -27,7 +27,7 @@ namespace ngraph
 class ngraph::pass::AssignTensors : public CallGraphPass
 {
 public:
-    virtual bool run_on_call_graph(std::list<Node*>& nodes) override;
+    virtual bool run_on_call_graph(std::list<std::shared_ptr<Node>>& nodes) override;
 
 private:
 };

src/ngraph/pass/collect_functions.cpp

@@ -24,22 +24,22 @@ using namespace std;
 using namespace ngraph;
 using namespace ngraph::pass;
 
-bool CollectFunctions::run_on_function(ngraph::Function* func)
+bool CollectFunctions::run_on_function(shared_ptr<ngraph::Function> func)
 {
-    set<Function*> functions;
-    deque<Function*> stack;
+    set<shared_ptr<ngraph::Function>> functions;
+    deque<shared_ptr<ngraph::Function>> stack;
     stack.push_back(func);
 
     while (stack.empty() == false)
     {
-        Function* f = stack.front();
+        shared_ptr<ngraph::Function> f = stack.front();
         stack.pop_front();
         functions.insert(f);
-        traverse_nodes(f->get_result(), [&](Node* node) {
-            op::FunctionCall* fc = dynamic_cast<op::FunctionCall*>(node);
+        traverse_nodes(f, [&](shared_ptr<Node> node) {
+            shared_ptr<op::FunctionCall> fc = dynamic_pointer_cast<op::FunctionCall>(node);
             if (fc)
             {
-                stack.push_back(fc->get_function().get());
+                stack.push_back(fc->get_function());
             }
         });
     }

src/ngraph/pass/collect_functions.hpp

@@ -27,7 +27,7 @@ namespace ngraph
 class ngraph::pass::CollectFunctions : public FunctionPass
 {
 public:
-    bool run_on_function(ngraph::Function*) override;
+    bool run_on_function(std::shared_ptr<ngraph::Function>) override;
 
 private:
 };

src/ngraph/pass/dump_sorted.cpp

@@ -28,14 +28,14 @@ pass::DumpSorted::DumpSorted(const string& output_file)
 {
 }
 
-bool pass::DumpSorted::run_on_module(vector<Function*>& functions)
+bool pass::DumpSorted::run_on_module(vector<shared_ptr<ngraph::Function>>& functions)
 {
     ofstream out{m_output_file};
     if (out)
     {
-        for (Function* f : functions)
+        for (shared_ptr<Function> f : functions)
         {
-            for (const Node* node : f->get_ordered_ops())
+            for (const shared_ptr<Node>& node : f->get_ordered_ops())
             {
                 out << node->get_name() << "(";
                 vector<string> inputs;

src/ngraph/pass/dump_sorted.hpp

@@ -31,7 +31,7 @@ class ngraph::pass::DumpSorted : public ModulePass
 public:
     DumpSorted(const std::string& output_file);
 
-    virtual bool run_on_module(std::vector<Function*>&) override;
+    virtual bool run_on_module(std::vector<std::shared_ptr<ngraph::Function>>&) override;
 
 private:
     const std::string m_output_file;

src/ngraph/pass/liveness.cpp

@@ -28,13 +28,13 @@ using namespace std;
 using namespace ngraph;
 using namespace ngraph::descriptor;
 
-bool pass::Liveness::run_on_call_graph(list<Node*>& ops)
+bool pass::Liveness::run_on_call_graph(list<shared_ptr<Node>>& ops)
 {
     unordered_set<Tensor*> currently_live;
 
     for (auto it = ops.rbegin(); it != ops.rend(); it++)
     {
-        Node* node = *it;
+        shared_ptr<Node> node = *it;
         node->liveness_live_list.clear();
         node->liveness_new_list.clear();
         node->liveness_free_list.clear();
@@ -91,7 +91,7 @@ bool pass::Liveness::run_on_call_graph(list<Node*>& ops)
     // Add outputs to live_list and remove from free_list
     unordered_set<Tensor*> outputs;
     unordered_set<Tensor*> seen;
-    for (Node* node : ops)
+    for (shared_ptr<Node> node : ops)
     {
         for (Tensor* tensor : node->liveness_live_list)
         {

src/ngraph/pass/liveness.hpp

@@ -28,7 +28,7 @@ namespace ngraph
 class ngraph::pass::Liveness : public CallGraphPass
 {
 public:
-    virtual bool run_on_call_graph(std::list<Node*>&) override;
+    virtual bool run_on_call_graph(std::list<std::shared_ptr<Node>>&) override;
 
 private:
     bool is_temporary(const descriptor::Tensor&);

src/ngraph/pass/manager.cpp

@@ -38,12 +38,7 @@ void ngraph::pass::Manager::initialize_default_passes()
 
 void ngraph::pass::Manager::run_passes(shared_ptr<Function> func)
 {
-    run_passes(func.get());
-}
-
-void ngraph::pass::Manager::run_passes(Function* func)
-{
-    vector<Function*> fs = {func};
+    vector<shared_ptr<Function>> fs = {func};
     get_state().set_functions(fs);
 
     for (shared_ptr<PassBase> pass : m_pass_list)
@@ -59,16 +54,16 @@ void ngraph::pass::Manager::run_passes(Function* func)
         }
         else if (function_pass)
         {
-            for (Function* f : fs)
+            for (shared_ptr<Function> f : fs)
             {
                 function_pass->run_on_function(f);
             }
         }
         else if (node_pass)
         {
-            for (Function* f : fs)
+            for (shared_ptr<Function> f : fs)
             {
-                for (Node* n : f->get_ops())
+                for (shared_ptr<Node> n : f->get_ops())
                 {
                     node_pass->run_on_node(n);
                 }
@@ -76,7 +71,7 @@ void ngraph::pass::Manager::run_passes(Function* func)
         }
         else if (call_graph_pass)
         {
-            for (Function* f : fs)
+            for (shared_ptr<Function> f : fs)
             {
                 call_graph_pass->run_on_call_graph(f->get_ordered_ops());
             }

src/ngraph/pass/manager.hpp

@@ -47,7 +47,6 @@ public:
         m_pass_list.push_back(pass_base);
     }
 
-    void run_passes(Function*);
     void run_passes(std::shared_ptr<Function>);
 
     ManagerState& get_state();

src/ngraph/pass/manager_state.cpp

@@ -23,7 +23,7 @@
 using namespace std;
 using namespace ngraph;
 
-vector<Function*>& ngraph::pass::ManagerState::get_functions()
+vector<shared_ptr<Function>>& ngraph::pass::ManagerState::get_functions()
 {
     return m_function_list;
 }

src/ngraph/pass/manager_state.hpp

@@ -30,7 +30,7 @@ namespace ngraph
 class ngraph::pass::ManagerState
 {
 public:
-    std::vector<Function*>& get_functions();
+    std::vector<std::shared_ptr<Function>>& get_functions();
 
     template <typename T>
     void set_functions(const T& collection)
@@ -44,5 +44,5 @@ public:
 
 private:
     size_t m_temporary_pool_size = 0;
-    std::vector<Function*> m_function_list;
+    std::vector<std::shared_ptr<Function>> m_function_list;
 };

src/ngraph/pass/memory_layout.cpp

@@ -26,10 +26,10 @@ using namespace std;
 using namespace ngraph;
 using namespace ngraph::descriptor;
 
-bool pass::MemoryLayout::run_on_call_graph(std::list<Node*>& node_list)
+bool pass::MemoryLayout::run_on_call_graph(std::list<std::shared_ptr<Node>>& node_list)
 {
     MemoryManager mm;
-    for (const Node* node : node_list)
+    for (shared_ptr<Node> node : node_list)
     {
         for (Tensor* tensor : node->liveness_new_list)
         {

src/ngraph/pass/memory_layout.hpp

@@ -33,7 +33,7 @@ namespace ngraph
 class ngraph::pass::MemoryLayout : public CallGraphPass
 {
 public:
-    virtual bool run_on_call_graph(std::list<Node*>&) override;
+    virtual bool run_on_call_graph(std::list<std::shared_ptr<Node>>&) override;
 
 private:
 };

src/ngraph/pass/memory_visualize.cpp

@@ -32,13 +32,13 @@ pass::MemoryVisualize::MemoryVisualize(const string& filename)
 {
 }
 
-bool pass::MemoryVisualize::run_on_module(vector<Function*>& functions)
+bool pass::MemoryVisualize::run_on_module(vector<shared_ptr<ngraph::Function>>& functions)
 {
     ofstream file(m_filename);
     {
-        for (const Function* f : functions)
+        for (shared_ptr<Function> f : functions)
         {
-            const list<Node*> nodes = f->get_ordered_ops();
+            list<shared_ptr<Node>> nodes = f->get_ordered_ops();
             file << "<!DOCTYPE html>\n<html>\n";
             file << "<head>\n";
             file << "    <style>\n";
@@ -62,7 +62,7 @@ bool pass::MemoryVisualize::run_on_module(vector<Function*>& functions)
             file << "<body>\n";
             unordered_set<descriptor::Tensor*> tensors;
             size_t temp_max_size = 0;
-            for (Node* node : nodes)
+            for (shared_ptr<Node> node : nodes)
             {
                 tensors.insert(node->liveness_live_list.begin(), node->liveness_live_list.end());
             }
@@ -96,11 +96,11 @@ bool pass::MemoryVisualize::run_on_module(vector<Function*>& functions)
     return false;
 }
 
-const Node* pass::MemoryVisualize::find_largest_op(const list<Node*>& nodes)
+shared_ptr<Node> pass::MemoryVisualize::find_largest_op(const list<shared_ptr<Node>>& nodes)
 {
-    const Node* largest_op = nullptr;
+    shared_ptr<Node> largest_op = nullptr;
     size_t largest_size = 0;
-    for (const Node* exop : nodes)
+    for (shared_ptr<Node> exop : nodes)
     {
         size_t size = 0;
         for (const Tensor* tensor : exop->liveness_live_list)
@@ -116,9 +116,9 @@ const Node* pass::MemoryVisualize::find_largest_op(const list<Node*>& nodes)
     return largest_op;
 }
 
-void pass::MemoryVisualize::draw_tensor_weight(ostream& file, const list<Node*>& nodes)
+void pass::MemoryVisualize::draw_tensor_weight(ostream& file, const list<shared_ptr<Node>>& nodes)
 {
-    const Node* largest_op = find_largest_op(nodes);
+    shared_ptr<Node> largest_op = find_largest_op(nodes);
 
     if (largest_op)
     {
@@ -130,7 +130,7 @@ void pass::MemoryVisualize::draw_tensor_weight(ostream& file, const list<Node*>&
 
         unordered_map<const Tensor*, size_t> age_list;
         vector<const Tensor*> tensor_set;
-        unordered_map<const Tensor*, const Node*> generator_op;
+        unordered_map<const Tensor*, shared_ptr<Node>> generator_op;
         file << "<table>\n";
         file << "    <tr>";
         file << "<th align=\"left\">tensor</th>";
@@ -139,7 +139,7 @@ void pass::MemoryVisualize::draw_tensor_weight(ostream& file, const list<Node*>&
         file << "<th align=\"right\">generator weight</th>";
         file << "</tr>\n";
         size_t i = 0;
-        for (const Node* exop : nodes)
+        for (shared_ptr<Node> exop : nodes)
         {
             for (const Tensor* tensor : exop->liveness_new_list)
             {
@@ -179,7 +179,7 @@ void pass::MemoryVisualize::draw_tensor_weight(ostream& file, const list<Node*>&
     }
 }
 
-void pass::MemoryVisualize::draw_histogram(ostream& file, const list<Node*>& nodes)
+void pass::MemoryVisualize::draw_histogram(ostream& file, const list<shared_ptr<Node>>& nodes)
 {
     size_t stroke_width = 14;
     size_t text_offset = 4;
@@ -188,7 +188,7 @@ void pass::MemoryVisualize::draw_histogram(ostream& file, const list<Node*>& nod
     size_t scale = width - offset;
     size_t line_spacing = stroke_width * 1.5;
     size_t line_count = 0;
-    for (const Node* node : nodes)
+    for (shared_ptr<Node> node : nodes)
     {
         (void)node;
         line_count += 1;
@@ -198,7 +198,7 @@ void pass::MemoryVisualize::draw_histogram(ostream& file, const list<Node*>& nod
 
     file << "<svg viewBox=\"0 0 " << width << " " << height << "\">\n";
     size_t y = 0;
-    for (const Node* node : nodes)
+    for (shared_ptr<Node> node : nodes)
     {
         float usage = float(MemoryVisualize::memory_usage(node));
         float footprint = float(MemoryVisualize::memory_footprint(node));
@@ -220,14 +220,14 @@ void pass::MemoryVisualize::draw_histogram(ostream& file, const list<Node*>& nod
     file << "</svg>\n";
 }
 
-void pass::MemoryVisualize::draw_op_influence(ostream& file, const list<Node*>& nodes)
+void pass::MemoryVisualize::draw_op_influence(ostream& file, const list<shared_ptr<Node>>& nodes)
 {
     file << "<table>\n";
     file << "    <tr>";
     file << "<th align=\"left\">op</th>";
     file << "<th align=\"right\">influence</th>";
     file << "</tr>\n";
-    for (const Node* exop : nodes)
+    for (shared_ptr<Node> exop : nodes)
     {
         int weight = compute_op_weight(exop);
         file << "    <tr>";
@@ -237,7 +237,7 @@ void pass::MemoryVisualize::draw_op_influence(ostream& file, const list<Node*>&
     }
 }
 
-int pass::MemoryVisualize::compute_op_weight(const Node* exop)
+int pass::MemoryVisualize::compute_op_weight(const shared_ptr<Node> exop)
 {
     int mass = 0;
     // for input_decl in exop.input_decls:
@@ -265,17 +265,17 @@ int pass::MemoryVisualize::compute_op_weight(const Node* exop)
     return mass;
 }
 
-size_t pass::MemoryVisualize::memory_usage(const Node* node)
+size_t pass::MemoryVisualize::memory_usage(shared_ptr<Node> node)
 {
     return 0;
 }
 
-size_t pass::MemoryVisualize::memory_footprint(const Node* node)
+size_t pass::MemoryVisualize::memory_footprint(shared_ptr<Node> node)
 {
     return 0;
 }
 
-size_t pass::MemoryVisualize::memory_footprint(const std::list<Node*>& nodes)
+size_t pass::MemoryVisualize::memory_footprint(const std::list<shared_ptr<Node>>& nodes)
 {
     return 0;
 }

src/ngraph/pass/memory_visualize.hpp

@@ -32,18 +32,18 @@ class ngraph::pass::MemoryVisualize : public ModulePass
 {
 public:
     MemoryVisualize(const std::string& filename);
-    virtual bool run_on_module(std::vector<Function*>&) override;
+    virtual bool run_on_module(std::vector<std::shared_ptr<ngraph::Function>>&) override;
 
 private:
-    const Node* find_largest_op(const std::list<Node*>& nodes);
-    void draw_tensor_weight(std::ostream& file, const std::list<Node*>& nodes);
-    void draw_histogram(std::ostream& file, const std::list<Node*>& nodes);
-    void draw_op_influence(std::ostream& file, const std::list<Node*>& nodes);
-    int compute_op_weight(const Node* exop);
-
-    static size_t memory_usage(const Node*);
-    static size_t memory_footprint(const Node*);
-    static size_t memory_footprint(const std::list<Node*>&);
+    std::shared_ptr<Node> find_largest_op(const std::list<std::shared_ptr<Node>>& nodes);
+    void draw_tensor_weight(std::ostream& file, const std::list<std::shared_ptr<Node>>& nodes);
+    void draw_histogram(std::ostream& file, const std::list<std::shared_ptr<Node>>& nodes);
+    void draw_op_influence(std::ostream& file, const std::list<std::shared_ptr<Node>>& nodes);
+    int compute_op_weight(std::shared_ptr<Node> exop);
+
+    static size_t memory_usage(std::shared_ptr<Node>);
+    static size_t memory_footprint(std::shared_ptr<Node>);
+    static size_t memory_footprint(const std::list<std::shared_ptr<Node>>&);
 
     const std::string m_filename;
 };

src/ngraph/pass/pass.hpp

@@ -53,26 +53,26 @@ class ngraph::pass::ModulePass : public PassBase
 {
 public:
     virtual ~ModulePass() {}
-    virtual bool run_on_module(std::vector<ngraph::Function*>&) = 0;
+    virtual bool run_on_module(std::vector<std::shared_ptr<ngraph::Function>>&) = 0;
 };
 
 class ngraph::pass::FunctionPass : public PassBase
 {
 public:
     virtual ~FunctionPass() {}
-    virtual bool run_on_function(ngraph::Function*) = 0;
+    virtual bool run_on_function(std::shared_ptr<ngraph::Function>) = 0;
 };
 
 class ngraph::pass::NodePass : public PassBase
 {
 public:
     virtual ~NodePass() {}
-    virtual bool run_on_node(ngraph::Node*) = 0;
+    virtual bool run_on_node(std::shared_ptr<ngraph::Node>) = 0;
 };
 
 class ngraph::pass::CallGraphPass : public PassBase
 {
 public:
     virtual ~CallGraphPass() {}
-    virtual bool run_on_call_graph(std::list<ngraph::Node*>&) = 0;
+    virtual bool run_on_call_graph(std::list<std::shared_ptr<ngraph::Node>>&) = 0;
 };

src/ngraph/pass/propagate_types.cpp

@@ -20,9 +20,9 @@
 using namespace std;
 using namespace ngraph;
 
-bool pass::PropagateTypes::run_on_call_graph(list<Node*>& nodes)
+bool pass::PropagateTypes::run_on_call_graph(list<shared_ptr<Node>>& nodes)
 {
-    for (Node* node : nodes)
+    for (shared_ptr<Node> node : nodes)
     {
         try
         {