More proper OpneCv+Qt+Vtk issues handling (http://code.opencv.org/issues/3492)

cmake/OpenCVDetectVTK.cmake

@@ -2,25 +2,52 @@ if(NOT WITH_VTK OR ANDROID OR IOS)
   return()
 endif()
 
-if (HAVE_QT5)
-  message(STATUS "VTK is disabled because OpenCV is linked with Q5. Some VTK disributives are compiled with Q4 and therefore can't be linked together Qt5.")
+# VTK 6.x components
+find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE)
+
+# VTK 5.x components
+if(NOT VTK_FOUND)
+  find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE)
+endif()
+
+if(NOT VTK_FOUND)
+  set(HAVE_VTK OFF)
+  message(STATUS "VTK is not found. Please set -DVTK_DIR in CMake to VTK build directory, or to VTK install subdirectory with VTKConfig.cmake file")
   return()
 endif()
 
-find_package(VTK 6.0 QUIET COMPONENTS vtkRenderingCore vtkInteractionWidgets vtkInteractionStyle vtkIOLegacy vtkIOPLY vtkRenderingFreeType vtkRenderingLOD vtkFiltersTexture vtkIOExport NO_MODULE)
+# Don't support ealier VTKs
+if(${VTK_VERSION} VERSION_LESS "5.8.0")
+  message(STATUS "VTK support is disabled. VTK ver. 5.8.0 is minimum required, but found VTK ver. ${VTK_VERSION}")
+  return()
+endif()
 
-if(NOT DEFINED VTK_FOUND OR NOT VTK_FOUND)
-  find_package(VTK 5.10 QUIET COMPONENTS vtkCommon vtkFiltering vtkRendering vtkWidgets vtkImaging NO_MODULE)
+# Different Qt versions can't be linked together
+if(HAVE_QT5 AND ${VTK_VERSION} VERSION_LESS "6.0.0")
+  if(VTK_USE_QT)
+    message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4")
+  endif()
 endif()
 
-if(NOT DEFINED VTK_FOUND OR NOT VTK_FOUND)
-  find_package(VTK 5.8 QUIET COMPONENTS vtkCommon vtkFiltering vtkRendering vtkWidgets vtkImaging NO_MODULE)
+# Different Qt versions can't be linked together. VTK 6.0.0 doesn't provide a way to get Qt version it was linked with
+if(HAVE_QT5 AND ${VTK_VERSION} VERSION_EQUAL "6.0.0" AND NOT DEFINED FORCE_VTK)
+  message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt5, and VTK ver.${VTK_VERSION} with Qt4")
+  message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DFORCE_VTK=TRUE' flag in CMake")
+  return()
 endif()
 
-if(VTK_FOUND)
-  set(HAVE_VTK ON)
-  message(STATUS "Found VTK ver. ${VTK_VERSION} (usefile: ${VTK_USE_FILE})")
-else()
-  set(HAVE_VTK OFF)
-  message(STATUS "VTK is not found. Please set -DVTK_DIR in CMake to VTK build directory, or set $VTK_DIR enviroment variable to VTK install subdirectory with VTKConfig.cmake file (for windows)")
+# Different Qt versions can't be linked together
+if(HAVE_QT AND ${VTK_VERSION} VERSION_GREATER "6.0.0" AND NOT ${VTK_QT_VERSION} STREQUAL "")
+  if(HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "4")
+    message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4")
+    return()
+  endif()
+
+  if(NOT HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "5")
+    message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt4 and VTK ver.${VTK_VERSION} + Qt5")
+    return()
+  endif()
 endif()
+
+set(HAVE_VTK ON)
+message(STATUS "Found VTK ver. ${VTK_VERSION} (usefile: ${VTK_USE_FILE})")