diff --git a/src/contrib/mlir/CMakeLists.txt b/src/contrib/mlir/CMakeLists.txt
index e11a54bf92a1e7b3246f9007f67e3dec9e92a7e3..7622d35206663ef6be0a8710270374b325c7fb31 100644
--- a/src/contrib/mlir/CMakeLists.txt
+++ b/src/contrib/mlir/CMakeLists.txt
@@ -14,103 +14,4 @@
 # limitations under the License.
 # ******************************************************************************
 
-set(SRC
-    dialect/dialect.cpp
-    dialect/type.cpp
-    dialect/ops.cpp
-    compiler.cpp
-    lowerer.cpp
-    memory_manager.cpp
-    pass/mlir_subgraph_extraction.cpp
-    pass/mlir_subgraph_extraction.hpp
-)
-if (NGRAPH_MLIR_ENABLE)
-
-    add_library(mlir_backend SHARED ${SRC})
-
-    message(STATUS "LLVM Directory: ${LLVM_DIR}")
-    # Link LLVM and MLIR
-    find_package(LLVM REQUIRED CONFIG)
-    
-    set(MLIR_LLVM_INCLUDEPATH ${LLVM_INCLUDE_DIRS})
-
-    message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
-    message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
-    message(STATUS "LLVM RTTI is ${LLVM_ENABLE_RTTI}")
-    add_definitions(${LLVM_DEFINITIONS})
-   
-    target_include_directories(mlir_backend PRIVATE ${LLVM_INCLUDE_DIRS})
-    
-    message(STATUS "MLIR Headers at : ${MLIR_INCLUDE_PATHS}")
-    message(STATUS "LLVM Headers at : ${MLIR_LLVM_INCLUDEPATH}")
-
-    target_include_directories(mlir_backend PRIVATE ${MLIR_INCLUDE_PATHS}) 
-    
-    llvm_map_components_to_libnames(llvm_libs support core irreader)
-    
-    # Link MLIR libs
-    target_link_libraries(
-        mlir_backend PRIVATE 
-        MLIRAnalysis
-        MLIREDSC
-        MLIRExecutionEngine
-        MLIRIR
-        MLIRLLVMIR
-        MLIRStandardToLLVM
-        MLIRParser
-        MLIRPass
-        MLIRTargetLLVMIR
-        MLIRTransforms
-        MLIRSupport
-    )
-    # some libs need whole archive linkage because of Globals static initialization
-    function(whole_archive_link target)
-        if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
-            set(link_flags "-Llib -Wl,-all_load ")
-            FOREACH(LIB ${ARGN})
-                string(CONCAT link_flags ${link_flags} "${LIB}")
-            ENDFOREACH(LIB)
-        else()
-            set(link_flags "-Llib -Wl,--whole-archive,")
-            FOREACH(LIB ${ARGN})
-                string(CONCAT link_flags ${link_flags} "${LIB},")
-            ENDFOREACH(LIB)
-            string(CONCAT link_flags ${link_flags} "--no-whole-archive")
-        endif()
-        message(STATUS "MLIR Ops link flag: ${link_flags}" )
-
-        set_target_properties(${target} PROPERTIES LINK_FLAGS ${link_flags})
-    endfunction(whole_archive_link)
-
-    whole_archive_link(mlir_backend
-        ${LLVM_BUILD_LIBRARY_DIR}/libMLIRAffineOps.a
-        ${LLVM_BUILD_LIBRARY_DIR}/libMLIRStandardOps.a
-    )
-    # Link LLVM libs
-    target_link_libraries(
-        mlir_backend PRIVATE 
-        ${llvm_libs}
-    )
-
-    # Link ngraph 
-    target_link_libraries(mlir_backend PUBLIC ngraph)
-
-    # table-gen dialect ops
-    # include table-gen helpers
-    include(${LLVM_DIR}/TableGen.cmake)
-
-    function(ngraph_tablegen ofn)
-        tablegen(MLIR ${ARGV} "-I${MLIR_SRC_INCLUDE_PATH}" "-I${MLIR_BIN_INCLUDE_PATH}")
-        set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn} PARENT_SCOPE)
-    endfunction()
-
-    set(MLIR_TABLEGEN_EXE mlir-tblgen)
-    set(LLVM_TARGET_DEFINITIONS dialect/ops.td)
-
-    ngraph_tablegen(ops.h.inc -gen-op-decls)
-    ngraph_tablegen(ops.cpp.inc -gen-op-defs)
-
-    add_public_tablegen_target(ngraph_ops_gen)
-    add_dependencies(mlir_backend ngraph_ops_gen)
-    target_include_directories(mlir_backend PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-endif()
+add_subdirectory(compiler)
diff --git a/src/contrib/mlir/compiler/CMakeLists.txt b/src/contrib/mlir/compiler/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ed92cb0f3305376f6e06b416adf4e4244d5e3098
--- /dev/null
+++ b/src/contrib/mlir/compiler/CMakeLists.txt
@@ -0,0 +1,114 @@
+# ******************************************************************************
+# Copyright 2017-2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ******************************************************************************
+
+set(SRC
+    dialect/dialect.cpp
+    dialect/type.cpp
+    dialect/ops.cpp
+    compiler.cpp
+    lowerer.cpp
+    memory_manager.cpp
+    pass/mlir_subgraph_extraction.cpp
+    pass/mlir_subgraph_extraction.hpp
+)
+
+add_library(mlir_backend SHARED ${SRC})
+
+message(STATUS "LLVM Directory: ${LLVM_DIR}")
+# Link LLVM and MLIR
+find_package(LLVM REQUIRED CONFIG)
+
+set(MLIR_LLVM_INCLUDEPATH ${LLVM_INCLUDE_DIRS})
+
+message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+message(STATUS "LLVM RTTI is ${LLVM_ENABLE_RTTI}")
+add_definitions(${LLVM_DEFINITIONS})
+
+target_include_directories(mlir_backend PRIVATE ${LLVM_INCLUDE_DIRS})
+
+message(STATUS "MLIR Headers at : ${MLIR_INCLUDE_PATHS}")
+message(STATUS "LLVM Headers at : ${MLIR_LLVM_INCLUDEPATH}")
+
+target_include_directories(mlir_backend PRIVATE ${MLIR_INCLUDE_PATHS}) 
+
+llvm_map_components_to_libnames(llvm_libs support core irreader)
+
+# Link MLIR libs
+target_link_libraries(
+    mlir_backend PRIVATE 
+    MLIRAnalysis
+    MLIREDSC
+    MLIRExecutionEngine
+    MLIRIR
+    MLIRLLVMIR
+    MLIRStandardToLLVM
+    MLIRParser
+    MLIRPass
+    MLIRTargetLLVMIR
+    MLIRTransforms
+    MLIRSupport
+)
+# some libs need whole archive linkage because of Globals static initialization
+function(whole_archive_link target)
+    if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
+        set(link_flags "-Llib -Wl,-all_load ")
+        FOREACH(LIB ${ARGN})
+            string(CONCAT link_flags ${link_flags} "${LIB}")
+        ENDFOREACH(LIB)
+    else()
+        set(link_flags "-Llib -Wl,--whole-archive,")
+        FOREACH(LIB ${ARGN})
+            string(CONCAT link_flags ${link_flags} "${LIB},")
+        ENDFOREACH(LIB)
+        string(CONCAT link_flags ${link_flags} "--no-whole-archive")
+    endif()
+    message(STATUS "MLIR Ops link flag: ${link_flags}" )
+
+    set_target_properties(${target} PROPERTIES LINK_FLAGS ${link_flags})
+endfunction(whole_archive_link)
+
+whole_archive_link(mlir_backend
+    ${LLVM_BUILD_LIBRARY_DIR}/libMLIRAffineOps.a
+    ${LLVM_BUILD_LIBRARY_DIR}/libMLIRStandardOps.a
+)
+# Link LLVM libs
+target_link_libraries(
+    mlir_backend PRIVATE 
+    ${llvm_libs}
+)
+
+# Link ngraph 
+target_link_libraries(mlir_backend PUBLIC ngraph)
+
+# table-gen dialect ops
+# include table-gen helpers
+include(${LLVM_DIR}/TableGen.cmake)
+
+function(ngraph_tablegen ofn)
+    tablegen(MLIR ${ARGV} "-I${MLIR_SRC_INCLUDE_PATH}" "-I${MLIR_BIN_INCLUDE_PATH}")
+    set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn} PARENT_SCOPE)
+endfunction()
+
+set(MLIR_TABLEGEN_EXE mlir-tblgen)
+set(LLVM_TARGET_DEFINITIONS dialect/ops.td)
+
+ngraph_tablegen(ops.h.inc -gen-op-decls)
+ngraph_tablegen(ops.cpp.inc -gen-op-defs)
+
+add_public_tablegen_target(ngraph_ops_gen)
+add_dependencies(mlir_backend ngraph_ops_gen)
+target_include_directories(mlir_backend PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/src/contrib/mlir/compiler.cpp b/src/contrib/mlir/compiler/compiler.cpp
similarity index 100%
rename from src/contrib/mlir/compiler.cpp
rename to src/contrib/mlir/compiler/compiler.cpp
diff --git a/src/contrib/mlir/compiler.hpp b/src/contrib/mlir/compiler/compiler.hpp
similarity index 100%
rename from src/contrib/mlir/compiler.hpp
rename to src/contrib/mlir/compiler/compiler.hpp
diff --git a/src/contrib/mlir/dialect/dialect.cpp b/src/contrib/mlir/compiler/dialect/dialect.cpp
similarity index 100%
rename from src/contrib/mlir/dialect/dialect.cpp
rename to src/contrib/mlir/compiler/dialect/dialect.cpp
diff --git a/src/contrib/mlir/dialect/dialect.hpp b/src/contrib/mlir/compiler/dialect/dialect.hpp
similarity index 100%
rename from src/contrib/mlir/dialect/dialect.hpp
rename to src/contrib/mlir/compiler/dialect/dialect.hpp
diff --git a/src/contrib/mlir/dialect/ops.cpp b/src/contrib/mlir/compiler/dialect/ops.cpp
similarity index 100%
rename from src/contrib/mlir/dialect/ops.cpp
rename to src/contrib/mlir/compiler/dialect/ops.cpp
diff --git a/src/contrib/mlir/dialect/ops.hpp b/src/contrib/mlir/compiler/dialect/ops.hpp
similarity index 100%
rename from src/contrib/mlir/dialect/ops.hpp
rename to src/contrib/mlir/compiler/dialect/ops.hpp
diff --git a/src/contrib/mlir/dialect/ops.td b/src/contrib/mlir/compiler/dialect/ops.td
similarity index 100%
rename from src/contrib/mlir/dialect/ops.td
rename to src/contrib/mlir/compiler/dialect/ops.td
diff --git a/src/contrib/mlir/dialect/type.cpp b/src/contrib/mlir/compiler/dialect/type.cpp
similarity index 100%
rename from src/contrib/mlir/dialect/type.cpp
rename to src/contrib/mlir/compiler/dialect/type.cpp
diff --git a/src/contrib/mlir/dialect/type.hpp b/src/contrib/mlir/compiler/dialect/type.hpp
similarity index 100%
rename from src/contrib/mlir/dialect/type.hpp
rename to src/contrib/mlir/compiler/dialect/type.hpp
diff --git a/src/contrib/mlir/lowerer.cpp b/src/contrib/mlir/compiler/lowerer.cpp
similarity index 100%
rename from src/contrib/mlir/lowerer.cpp
rename to src/contrib/mlir/compiler/lowerer.cpp
diff --git a/src/contrib/mlir/lowerer.hpp b/src/contrib/mlir/compiler/lowerer.hpp
similarity index 96%
rename from src/contrib/mlir/lowerer.hpp
rename to src/contrib/mlir/compiler/lowerer.hpp
index fa97f4952aeb41bc104425a6ec17c91d8ad9e41e..ffe5eb65cfc114441051a42593c2372af0c9bce2 100644
--- a/src/contrib/mlir/lowerer.hpp
+++ b/src/contrib/mlir/compiler/lowerer.hpp
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "contrib/mlir/compiler.hpp"
+#include "contrib/mlir/compiler/compiler.hpp"
 
 #include <mlir/Pass/Pass.h>
 
diff --git a/src/contrib/mlir/memory_manager.cpp b/src/contrib/mlir/compiler/memory_manager.cpp
similarity index 100%
rename from src/contrib/mlir/memory_manager.cpp
rename to src/contrib/mlir/compiler/memory_manager.cpp
diff --git a/src/contrib/mlir/memory_manager.hpp b/src/contrib/mlir/compiler/memory_manager.hpp
similarity index 100%
rename from src/contrib/mlir/memory_manager.hpp
rename to src/contrib/mlir/compiler/memory_manager.hpp
diff --git a/src/contrib/mlir/op_lowerers.inc b/src/contrib/mlir/compiler/op_lowerers.inc
similarity index 100%
rename from src/contrib/mlir/op_lowerers.inc
rename to src/contrib/mlir/compiler/op_lowerers.inc
diff --git a/src/contrib/mlir/ops_supported.inc b/src/contrib/mlir/compiler/ops_supported.inc
similarity index 100%
rename from src/contrib/mlir/ops_supported.inc
rename to src/contrib/mlir/compiler/ops_supported.inc
diff --git a/src/contrib/mlir/pass/mlir_subgraph_extraction.cpp b/src/contrib/mlir/compiler/pass/mlir_subgraph_extraction.cpp
similarity index 99%
rename from src/contrib/mlir/pass/mlir_subgraph_extraction.cpp
rename to src/contrib/mlir/compiler/pass/mlir_subgraph_extraction.cpp
index 8541e9afacf0ff13a19b6b97a19c986841e74953..17b3ae61fc017e0ef628ee155c42a7d7e4d76ecb 100644
--- a/src/contrib/mlir/pass/mlir_subgraph_extraction.cpp
+++ b/src/contrib/mlir/compiler/pass/mlir_subgraph_extraction.cpp
@@ -533,5 +533,5 @@ bool MLIRSubgraphExtractionPass::check_cycles(std::shared_ptr<Node> node,
 
 const std::set<std::type_index> MLIRSubgraphExtractionPass::m_supported_ops{
 #define MLIR_OP(OP) TI(ngraph::op::OP),
-#include "contrib/mlir/ops_supported.inc"
+#include "contrib/mlir/compiler/ops_supported.inc"
 };
diff --git a/src/contrib/mlir/pass/mlir_subgraph_extraction.hpp b/src/contrib/mlir/compiler/pass/mlir_subgraph_extraction.hpp
similarity index 100%
rename from src/contrib/mlir/pass/mlir_subgraph_extraction.hpp
rename to src/contrib/mlir/compiler/pass/mlir_subgraph_extraction.hpp
diff --git a/src/ngraph/runtime/cpu/CMakeLists.txt b/src/ngraph/runtime/cpu/CMakeLists.txt
index 658db21b721ba36782eaa530241842c418de3719..5f868051555b0a975c07789031ae9277ae6b115d 100644
--- a/src/ngraph/runtime/cpu/CMakeLists.txt
+++ b/src/ngraph/runtime/cpu/CMakeLists.txt
@@ -229,7 +229,7 @@ if (NGRAPH_CPU_ENABLE)
         # TODO: can we get away without LLVM/MLIR include path. 
         # Currently mlir backend compiler.hpp include LLVM/MLIR files
         get_directory_property(MLIR_LLVM_INCLUDEPATH
-                               DIRECTORY ${NGRAPH_MLIR_SOURCE_DIR}
+                               DIRECTORY ${NGRAPH_MLIR_SOURCE_DIR}/compiler
                                DEFINITION MLIR_LLVM_INCLUDEPATH)
         
         message(STATUS "Building CPU backend with MLIR")
diff --git a/src/ngraph/runtime/cpu/builder/mlir_cpu_compiled_kernel.cpp b/src/ngraph/runtime/cpu/builder/mlir_cpu_compiled_kernel.cpp
index 248cda1ec92a10584e9e035fef02af0a265a5d38..0e29dfd81e8e72cf6ecdb2f3783cf0af11b4679c 100644
--- a/src/ngraph/runtime/cpu/builder/mlir_cpu_compiled_kernel.cpp
+++ b/src/ngraph/runtime/cpu/builder/mlir_cpu_compiled_kernel.cpp
@@ -16,7 +16,7 @@
 
 #include "ngraph/runtime/cpu/cpu_builder.hpp"
 
-#include "contrib/mlir/compiler.hpp"
+#include "contrib/mlir/compiler/compiler.hpp"
 #include "ngraph/op/experimental/compiled_kernel.hpp"
 #include "ngraph/runtime/cpu/cpu_runtime_context.hpp"
 
diff --git a/src/ngraph/runtime/cpu/cpu_backend.cpp b/src/ngraph/runtime/cpu/cpu_backend.cpp
index 8743658b07d7a76798898d29199f766af373296e..607878c1b2d532765deb68e41bb9ea98dfc79a37 100644
--- a/src/ngraph/runtime/cpu/cpu_backend.cpp
+++ b/src/ngraph/runtime/cpu/cpu_backend.cpp
@@ -29,7 +29,7 @@
 #include "ngraph/util.hpp"
 
 #ifdef NGRAPH_MLIR_ENABLE
-#include "contrib/mlir/compiler.hpp"
+#include "contrib/mlir/compiler/compiler.hpp"
 #endif
 
 using namespace ngraph;
diff --git a/src/ngraph/runtime/cpu/cpu_builder.cpp b/src/ngraph/runtime/cpu/cpu_builder.cpp
index b7937cabff5ac69ac4fffa88f73b8b2327a2e793..1200af609bc398f4e0cba37d447e5ea4c595d088 100644
--- a/src/ngraph/runtime/cpu/cpu_builder.cpp
+++ b/src/ngraph/runtime/cpu/cpu_builder.cpp
@@ -112,7 +112,7 @@
 #include "ngraph/util.hpp"
 
 #ifdef NGRAPH_MLIR_ENABLE
-#include "contrib/mlir/compiler.hpp"
+#include "contrib/mlir/compiler/compiler.hpp"
 #endif
 
 using namespace std;
diff --git a/src/ngraph/runtime/cpu/cpu_external_function.cpp b/src/ngraph/runtime/cpu/cpu_external_function.cpp
index 85534de88cdac43bac6bf286ea6d735acacef1d0..e8e59b64a3a6ebe892e7455ce4b533d6427bd9ed 100644
--- a/src/ngraph/runtime/cpu/cpu_external_function.cpp
+++ b/src/ngraph/runtime/cpu/cpu_external_function.cpp
@@ -34,7 +34,7 @@
 #endif
 
 #ifdef NGRAPH_MLIR_ENABLE
-#include "contrib/mlir/pass/mlir_subgraph_extraction.hpp"
+#include "contrib/mlir/compiler/pass/mlir_subgraph_extraction.hpp"
 #endif
 
 #include "ngraph/descriptor/input.hpp"
diff --git a/src/ngraph/runtime/cpu/cpu_runtime_context.hpp b/src/ngraph/runtime/cpu/cpu_runtime_context.hpp
index b241352c3f89d0d7ecbc20b657a0afd129f4b4d6..0289e8afd057b270585ac63a4739a773356ada12 100644
--- a/src/ngraph/runtime/cpu/cpu_runtime_context.hpp
+++ b/src/ngraph/runtime/cpu/cpu_runtime_context.hpp
@@ -28,7 +28,7 @@
 #include "ngraph/op/experimental/compiled_kernel.hpp"
 
 #ifdef NGRAPH_MLIR_ENABLE
-#include "contrib/mlir/compiler.hpp"
+#include "contrib/mlir/compiler/compiler.hpp"
 #endif
 
 namespace mkldnn