doc: finalize Python signatures injection

doc/CMakeLists.txt

@@ -218,7 +218,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
     COMPONENT "docs" OPTIONAL
   )
 
-  if(BUILD_opencv_python2)
+  if(PYTHON2_EXECUTABLE)
     add_custom_target(doxygen_python
       COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python"
       DEPENDS "${doxygen_result}" gen_opencv_python2
@@ -226,7 +226,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
     add_custom_target(doxygen
       DEPENDS doxygen_cpp doxygen_python
     )
-  elseif(BUILD_opencv_python3)
+  elseif(PYTHON3_EXECUTABLE)
     add_custom_target(doxygen_python
       COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python"
       DEPENDS "${doxygen_result}" gen_opencv_python3

doc/tools/add_signatures.py

@@ -12,10 +12,15 @@ TODO:
     http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41
 """
 from __future__ import print_function
+import sys
+sys.dont_write_bytecode = True  # Don't generate .pyc files / __pycache__ directories
+
 import os
+from pprint import pprint
 import re
-import sys
 import logging
+import json
+
 import html_functions
 import doxygen_scan
 
@@ -23,37 +28,23 @@ loglevel=os.environ.get("LOGLEVEL", None)
 if loglevel:
     logging.basicConfig(level=loglevel)
 
-
 ROOT_DIR = sys.argv[1]
 PYTHON_SIGNATURES_FILE = sys.argv[2]
-JAVA_PYTHON = sys.argv[3]
+JAVA_OR_PYTHON = sys.argv[3]
 
 ADD_JAVA = False
 ADD_PYTHON = False
-if JAVA_PYTHON == "python":
+if JAVA_OR_PYTHON == "python":
     ADD_PYTHON = True
 
-import json
 python_signatures = dict()
 with open(PYTHON_SIGNATURES_FILE, "rt") as f:
     python_signatures = json.load(f)
     print("Loaded Python signatures: %d" % len(python_signatures))
 
-# only name -> class
-# name and ret -> constant
-# name, ret, arg-> function / class method
-
-class Configuration():
-    def __init__(self):
-        self.ADD_PYTHON = ADD_PYTHON
-        self.python_signatures = python_signatures
-        self.ADD_JAVA = ADD_JAVA
-
-config = Configuration()
-
 import xml.etree.ElementTree as ET
 root = ET.parse(ROOT_DIR + 'opencv.tag')
-files_dict = dict()
+files_dict = {}
 
 # constants and function from opencv.tag
 namespaces = root.findall("./compound[@kind='namespace']")
@@ -61,41 +52,48 @@ namespaces = root.findall("./compound[@kind='namespace']")
 for ns in namespaces:
     ns_name = ns.find("./name").text
     #print('NS: {}'.format(ns_name))
-
-    files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
-    files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
+    doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
+    doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
 
 # class methods from opencv.tag
 classes = root.findall("./compound[@kind='class']")
 #print("Found {} classes".format(len(classes)))
 for c in classes:
     c_name = c.find("./name").text
-    name = ns_name + '::' + c_name
     file = c.find("./filename").text
-    #print('Class: {} => {}'.format(name, file))
-    files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict)
+    #print('Class: {} => {}'.format(c_name, file))
+    doxygen_scan.scan_class_methods(c, c_name, files_dict)
+
+print('Doxygen files to scan: %s' % len(files_dict))
+
+files_processed = 0
+files_skipped = 0
+symbols_processed = 0
 
-# test
 for file in files_dict:
-    soup = html_functions.load_html_file(ROOT_DIR + file)
-    if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html":
+    #if file != "dd/d9e/classcv_1_1VideoWriter.html":
+    #if file != "d4/d86/group__imgproc__filter.html":
+    #if file != "df/dfb/group__imgproc__object.html":
+    #    continue
+    #print('File: ' + file)
+
     anchor_list = files_dict[file]
-        counter = 0
-        anchor_tmp_list = []
-        for anchor in anchor_list:
-            counter += 1
-            # if the next anchor shares the same C++ name (= same method/function), join them together
-            if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname:
-                anchor_tmp_list.append(anchor)
+    active_anchors = [a for a in anchor_list if a.cppname in python_signatures]
+    if len(active_anchors) == 0: # no linked Python symbols
+        #print('Skip: ' + file)
+        files_skipped = files_skipped + 1
         continue
-            else:
-                anchor_tmp_list.append(anchor)
-            # check if extists a python equivalent signature
-            for signature in python_signatures: # signature is a key with the C++ name
-                if signature == anchor.cppname: # if available name in python
-                    # they should also have the same type
-                    soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup)
-                    #print(signature)
-            # reset anchor temporary list
-            anchor_tmp_list[:] = []
-        html_functions.update_html(ROOT_DIR + file, soup)
+
+    active_anchors_dict = {a.anchor: a for a in active_anchors}
+    if len(active_anchors_dict) != len(active_anchors):
+        logging.info('Duplicate entries detected: %s -> %s (%s)' % (len(active_anchors), len(active_anchors_dict), file))
+
+    files_processed = files_processed + 1
+
+    #pprint(active_anchors)
+    symbols_processed = symbols_processed + len(active_anchors_dict)
+
+    logging.info('File: %r' % file)
+    html_functions.insert_python_signatures(python_signatures, active_anchors_dict, ROOT_DIR + file)
+
+print('Done (processed files %d, symbols %d, skipped %d files)' % (files_processed, symbols_processed, files_skipped))

doc/tools/doxygen_scan.py

-class Anchor(object):
-    anchor = ""
-    type = ""
-    cppname = ""
+import traceback
 
+class Symbol(object):
     def __init__(self, anchor, type, cppname):
         self.anchor = anchor
         self.type = type
         self.cppname = cppname
+        #if anchor == 'ga586ebfb0a7fb604b35a23d85391329be':
+        #    print(repr(self))
+        #    traceback.print_stack()
+
+    def __repr__(self):
+        return '%s:%s@%s' % (self.type, self.cppname, self.anchor)
 
 def add_to_file(files_dict, file, anchor):
-    if file in files_dict:
-        # if that file already exists as a key in the dictionary
-        files_dict[file].append(anchor)
-    else:
-        files_dict[file] = [anchor]
-    return files_dict
+    anchors = files_dict.setdefault(file, [])
+    anchors.append(anchor)
 
 
 def scan_namespace_constants(ns, ns_name, files_dict):
@@ -25,8 +25,7 @@ def scan_namespace_constants(ns, ns_name, files_dict):
         file = c.find("./anchorfile").text
         anchor = c.find("./anchor").text
         #print('    CONST: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name))
-    return files_dict
+        add_to_file(files_dict, file, Symbol(anchor, "const", name))
 
 def scan_namespace_functions(ns, ns_name, files_dict):
     functions = ns.findall("./member[@kind='function']")
@@ -36,8 +35,7 @@ def scan_namespace_functions(ns, ns_name, files_dict):
         file = f.find("./anchorfile").text
         anchor = f.find("./anchor").text
         #print('    FN: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name))
-    return files_dict
+        add_to_file(files_dict, file, Symbol(anchor, "fn", name))
 
 def scan_class_methods(c, c_name, files_dict):
     methods = c.findall("./member[@kind='function']")
@@ -47,5 +45,4 @@ def scan_class_methods(c, c_name, files_dict):
         file = m.find("./anchorfile").text
         anchor = m.find("./anchor").text
         #print('    Method: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name))
-    return files_dict
+        add_to_file(files_dict, file, Symbol(anchor, "method", name))