doc: finalize Python signatures injection

pull/9260/head
Alexander Alekhin 7 years ago
parent 164a77e7ad
commit b15bc194ef
  1. 4
      doc/CMakeLists.txt
  2. 92
      doc/tools/add_signatures.py
  3. 29
      doc/tools/doxygen_scan.py
  4. 268
      doc/tools/html_functions.py

@ -218,7 +218,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
COMPONENT "docs" OPTIONAL
)
if(BUILD_opencv_python2)
if(PYTHON2_EXECUTABLE)
add_custom_target(doxygen_python
COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python"
DEPENDS "${doxygen_result}" gen_opencv_python2
@ -226,7 +226,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
add_custom_target(doxygen
DEPENDS doxygen_cpp doxygen_python
)
elseif(BUILD_opencv_python3)
elseif(PYTHON3_EXECUTABLE)
add_custom_target(doxygen_python
COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python"
DEPENDS "${doxygen_result}" gen_opencv_python3

@ -12,10 +12,15 @@ TODO:
http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41
"""
from __future__ import print_function
import sys
sys.dont_write_bytecode = True # Don't generate .pyc files / __pycache__ directories
import os
from pprint import pprint
import re
import sys
import logging
import json
import html_functions
import doxygen_scan
@ -23,37 +28,23 @@ loglevel=os.environ.get("LOGLEVEL", None)
if loglevel:
logging.basicConfig(level=loglevel)
ROOT_DIR = sys.argv[1]
PYTHON_SIGNATURES_FILE = sys.argv[2]
JAVA_PYTHON = sys.argv[3]
JAVA_OR_PYTHON = sys.argv[3]
ADD_JAVA = False
ADD_PYTHON = False
if JAVA_PYTHON == "python":
if JAVA_OR_PYTHON == "python":
ADD_PYTHON = True
import json
python_signatures = dict()
with open(PYTHON_SIGNATURES_FILE, "rt") as f:
python_signatures = json.load(f)
print("Loaded Python signatures: %d" % len(python_signatures))
# only name -> class
# name and ret -> constant
# name, ret, arg-> function / class method
class Configuration():
def __init__(self):
self.ADD_PYTHON = ADD_PYTHON
self.python_signatures = python_signatures
self.ADD_JAVA = ADD_JAVA
config = Configuration()
import xml.etree.ElementTree as ET
root = ET.parse(ROOT_DIR + 'opencv.tag')
files_dict = dict()
files_dict = {}
# constants and function from opencv.tag
namespaces = root.findall("./compound[@kind='namespace']")
@ -61,41 +52,48 @@ namespaces = root.findall("./compound[@kind='namespace']")
for ns in namespaces:
ns_name = ns.find("./name").text
#print('NS: {}'.format(ns_name))
files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
# class methods from opencv.tag
classes = root.findall("./compound[@kind='class']")
#print("Found {} classes".format(len(classes)))
for c in classes:
c_name = c.find("./name").text
name = ns_name + '::' + c_name
file = c.find("./filename").text
#print('Class: {} => {}'.format(name, file))
files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict)
#print('Class: {} => {}'.format(c_name, file))
doxygen_scan.scan_class_methods(c, c_name, files_dict)
print('Doxygen files to scan: %s' % len(files_dict))
files_processed = 0
files_skipped = 0
symbols_processed = 0
# test
for file in files_dict:
soup = html_functions.load_html_file(ROOT_DIR + file)
if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html":
anchor_list = files_dict[file]
counter = 0
anchor_tmp_list = []
for anchor in anchor_list:
counter += 1
# if the next anchor shares the same C++ name (= same method/function), join them together
if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname:
anchor_tmp_list.append(anchor)
continue
else:
anchor_tmp_list.append(anchor)
# check if extists a python equivalent signature
for signature in python_signatures: # signature is a key with the C++ name
if signature == anchor.cppname: # if available name in python
# they should also have the same type
soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup)
#print(signature)
# reset anchor temporary list
anchor_tmp_list[:] = []
html_functions.update_html(ROOT_DIR + file, soup)
#if file != "dd/d9e/classcv_1_1VideoWriter.html":
#if file != "d4/d86/group__imgproc__filter.html":
#if file != "df/dfb/group__imgproc__object.html":
# continue
#print('File: ' + file)
anchor_list = files_dict[file]
active_anchors = [a for a in anchor_list if a.cppname in python_signatures]
if len(active_anchors) == 0: # no linked Python symbols
#print('Skip: ' + file)
files_skipped = files_skipped + 1
continue
active_anchors_dict = {a.anchor: a for a in active_anchors}
if len(active_anchors_dict) != len(active_anchors):
logging.info('Duplicate entries detected: %s -> %s (%s)' % (len(active_anchors), len(active_anchors_dict), file))
files_processed = files_processed + 1
#pprint(active_anchors)
symbols_processed = symbols_processed + len(active_anchors_dict)
logging.info('File: %r' % file)
html_functions.insert_python_signatures(python_signatures, active_anchors_dict, ROOT_DIR + file)
print('Done (processed files %d, symbols %d, skipped %d files)' % (files_processed, symbols_processed, files_skipped))

@ -1,20 +1,20 @@
class Anchor(object):
anchor = ""
type = ""
cppname = ""
import traceback
class Symbol(object):
def __init__(self, anchor, type, cppname):
self.anchor = anchor
self.type = type
self.cppname = cppname
#if anchor == 'ga586ebfb0a7fb604b35a23d85391329be':
# print(repr(self))
# traceback.print_stack()
def __repr__(self):
return '%s:%s@%s' % (self.type, self.cppname, self.anchor)
def add_to_file(files_dict, file, anchor):
if file in files_dict:
# if that file already exists as a key in the dictionary
files_dict[file].append(anchor)
else:
files_dict[file] = [anchor]
return files_dict
anchors = files_dict.setdefault(file, [])
anchors.append(anchor)
def scan_namespace_constants(ns, ns_name, files_dict):
@ -25,8 +25,7 @@ def scan_namespace_constants(ns, ns_name, files_dict):
file = c.find("./anchorfile").text
anchor = c.find("./anchor").text
#print(' CONST: {} => {}#{}'.format(name, file, anchor))
files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name))
return files_dict
add_to_file(files_dict, file, Symbol(anchor, "const", name))
def scan_namespace_functions(ns, ns_name, files_dict):
functions = ns.findall("./member[@kind='function']")
@ -36,8 +35,7 @@ def scan_namespace_functions(ns, ns_name, files_dict):
file = f.find("./anchorfile").text
anchor = f.find("./anchor").text
#print(' FN: {} => {}#{}'.format(name, file, anchor))
files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name))
return files_dict
add_to_file(files_dict, file, Symbol(anchor, "fn", name))
def scan_class_methods(c, c_name, files_dict):
methods = c.findall("./member[@kind='function']")
@ -47,5 +45,4 @@ def scan_class_methods(c, c_name, files_dict):
file = m.find("./anchorfile").text
anchor = m.find("./anchor").text
#print(' Method: {} => {}#{}'.format(name, file, anchor))
files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name))
return files_dict
add_to_file(files_dict, file, Symbol(anchor, "method", name))

@ -1,7 +1,10 @@
from __future__ import print_function
import sys
import logging
import os
from pprint import pprint
import traceback
try:
import bs4
@ -13,195 +16,110 @@ except ImportError:
def load_html_file(file_dir):
""" Uses BeautifulSoup to load an html """
with open(file_dir) as fp:
with open(file_dir, 'rb') as fp:
soup = BeautifulSoup(fp, 'html.parser')
return soup
def add_item(soup, new_row, is_parameter, text):
""" Adds a new html tag for the table with the signature """
new_item = soup.new_tag('td')
if is_parameter:
new_item = soup.new_tag('td', **{'class': 'paramname'})
new_item.append(text)
new_row.append(new_item)
return new_row, soup
def add_signature_to_table(soup, tmp_row, signature, language, type):
""" Add a signature to an html table"""
new_item = soup.new_tag('td', style="padding-left: 0.5cm;")
def update_html(file, soup):
s = str(soup)
if os.name == 'nt' or sys.version_info[0] == 3: # if Windows
s = s.encode('utf-8', 'ignore')
with open(file, 'wb') as f:
f.write(s)
def insert_python_signatures(python_signatures, symbols_dict, filepath):
soup = load_html_file(filepath)
entries = soup.find_all(lambda tag: tag.name == "a" and tag.has_attr('id'))
for e in entries:
anchor = e['id']
if anchor in symbols_dict:
s = symbols_dict[anchor]
logging.info('Process: %r' % s)
if s.type == 'fn' or s.type == 'method':
process_fn(soup, e, python_signatures[s.cppname], s)
elif s.type == 'const':
process_const(soup, e, python_signatures[s.cppname], s)
else:
logging.error('unsupported type: %s' % s);
update_html(filepath, soup)
def process_fn(soup, anchor, python_signature, symbol):
try:
r = anchor.find_next_sibling(class_='memitem').find(class_='memproto').find('table')
insert_python_fn_signature(soup, r, python_signature, symbol)
except:
logging.error("Can't process: %s" % symbol)
traceback.print_exc()
pprint(anchor)
def process_const(soup, anchor, python_signature, symbol):
try:
#pprint(anchor.parent)
description = append(soup.new_tag('div', **{'class' : ['python_language']}),
'Python: ' + python_signature[0]['name'])
old = anchor.find_next_sibling('div', class_='python_language')
if old is None:
anchor.parent.append(description)
else:
old.replace_with(description)
#pprint(anchor.parent)
except:
logging.error("Can't process: %s" % symbol)
traceback.print_exc()
pprint(anchor)
if str(signature.get('ret', None)) != "None":
new_item.append(signature.get('ret') + ' =')
tmp_row.append(new_item)
tmp_name = signature.get('name', None)
if type is not "method":
tmp_name = "cv2." + tmp_name
else:
tmp_name = "obj." + tmp_name
tmp_row, soup = add_item(soup, tmp_row, False, tmp_name + '(')
tmp_row, soup = add_item(soup, tmp_row, True, signature['arg'])
tmp_row, soup = add_item(soup, tmp_row, False, ')')
return tmp_row, soup
def new_line(soup, tmp_table, new_row):
""" Adds a new line to the html table """
tmp_table.append(new_row)
new_row = soup.new_tag('tr')
return new_row, soup
def add_bolded(soup, new_row, text):
""" Adds bolded text to the table """
new_item = soup.new_tag('th', style="text-align:left")
new_item.append(text)
new_row.append(new_item)
return new_row, soup
def create_description(soup, language, signatures, type):
""" Insert the new Python / Java table after the current html c++ table """
assert signatures
tmp_table = soup.new_tag('table')
new_row = soup.new_tag('tr')
new_row, soup = add_bolded(soup, new_row, language)
new_row, soup = new_line(soup, tmp_table, new_row)
for s in signatures:
new_row, soup = new_line(soup, tmp_table, new_row)
new_row, soup = add_signature_to_table(soup, new_row, s, language, type)
new_row, soup = new_line(soup, tmp_table, new_row)
return tmp_table, soup
def get_anchor_list(anchor, soup):
a_list = []
# go through all the links
for a in soup.find_all('a', href=True):
# find links with the same anchor
last_part_of_link = a['href'].rsplit('#', 1)[-1]
if last_part_of_link == anchor:
a_list.append(a)
return a_list
def is_static_method(element):
if element.name == "table":
tmp_element = element.find('td', {'class': 'memname'})
if tmp_element is not None:
if 'static' in tmp_element.text:
return True
else:
if element['class'][0] == 'memItemRight':
if "static" in element.previousSibling.text:
return True
return False
def append_python_signatures_to_table(soup, signatures, table, type):
if type == "method":
if is_static_method(table):
type = "static" + type
description, soup = create_description(soup, "Python:", signatures, type)
def insert_python_fn_signature(soup, table, variants, symbol):
description = create_python_fn_description(soup, variants)
description['class'] = 'python_language'
soup = insert_or_replace(soup, table, description, "table", "python_language")
soup = insert_or_replace(table, description, 'table', 'python_language')
return soup
def get_heading_text(a):
str = ""
element = a.parent
if element is not None:
childs = element.find_all('a')
# the anchor should not be an argument of a function / method
if childs.index(a) is not 0:
return str
element = element.parent
if element is not None:
if element.has_attr('class'):
tmp_class = element["class"][0]
if "memitem:" in tmp_class and "python" not in tmp_class:
str = element.parent.find("tr").text
return str
def insert_or_replace(soup, element, description, tag_name, tag_class):
old = element.next_sibling
if old is not None:
if old.name != tag_name:
old = None
elif not tag_class in old.get('class', []):
old = None
# if already existed replace with the new
if old is None:
element.insert_after(description)
else:
old.replace_with(description)
return soup
def new_heading_td(soup, s, href, type):
if href is None:
attrs = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'}
new_td = soup.new_tag('td', **attrs)
new_td.append(str(s.get('ret', None)))
else:
attrs = {'class': 'memItemRight', 'valign': 'bottom'}
new_td = soup.new_tag('td', **attrs)
# make the function name linkable
attrs_a = {'class': 'el', 'href': href}
new_a = soup.new_tag('a', **attrs_a)
tmp_name = str(s.get('name', None))
if type is not "method":
tmp_name = "cv2." + tmp_name
else:
tmp_name = "obj." + tmp_name
new_a.append(tmp_name)
new_td.append(new_a)
def create_python_fn_description(soup, variants):
language = 'Python:'
table = soup.new_tag('table')
heading_row = soup.new_tag('th')
table.append(
append(soup.new_tag('tr'),
append(soup.new_tag('th', colspan=999, style="text-align:left"), language)))
for v in variants:
#logging.debug(v)
add_signature_to_table(soup, table, v, language, type)
#print(table)
return table
new_td.append("(" + s['arg'] +")")
return soup, new_td
def append_python_signatures_to_heading(soup, signatures, element, href, type):
if type == "method":
if is_static_method(element):
type = "static" + type
for s in signatures:
attrs = {'class': 'memitem:python'}
new_tr = soup.new_tag('tr', **attrs)
def add_signature_to_table(soup, table, signature, language, type):
""" Add a signature to an html table"""
row = soup.new_tag('tr')
row.append(soup.new_tag('td', style='width: 20px;'))
soup, new_td_left = new_heading_td(soup, s, None, type)
new_tr.append(new_td_left)
if 'ret' in signature:
row.append(append(soup.new_tag('td'), signature['ret']))
row.append(append(soup.new_tag('td'), '='))
else:
row.append(soup.new_tag('td')) # return values
row.append(soup.new_tag('td')) # '='
soup, new_td_right = new_heading_td(soup, s, href, type)
new_tr.append(new_td_right)
row.append(append(soup.new_tag('td'), signature['name'] + '('))
row.append(append(soup.new_tag('td', **{'class': 'paramname'}), signature['arg']))
row.append(append(soup.new_tag('td'), ')'))
table.append(row)
soup = insert_or_replace(soup, element, new_tr, "tr", "memitem:python")
return soup
def append_python_signature(function_variants, anchor_list, soup):
type = anchor_list[0].type
if type == "method" or type == "fn":
if len(anchor_list) == 1:
tmp_anchor = anchor_list[0].anchor
a_list = get_anchor_list(tmp_anchor, soup)
for a in a_list:
if a['href'] == "#" + tmp_anchor:
tmp_element = a.parent
# ignore the More... link <td class = mdescRight>
if tmp_element is None or tmp_element['class'][0] == 'mdescRight':
continue
# Function Documentation (tables)
table = a.findNext('table')
if table is not None:
soup = append_python_signatures_to_table(soup, function_variants, table, type)
else:
str = get_heading_text(a)
if "Functions" in str:
soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href'], type)
return soup
def append(target, obj):
target.append(obj)
return target
def update_html(file, soup):
tmp_str = str(soup)
if os.name == 'nt': # if Windows
with open(file, "wb") as tmp_file:
tmp_file.write(tmp_str.encode("ascii","ignore"))
def insert_or_replace(element_before, new_element, tag, tag_class):
old = element_before.find_next_sibling(tag, class_=tag_class)
if old is None:
element_before.insert_after(new_element)
else:
with open(file, "w") as tmp_file:
tmp_file.write(tmp_str)
old.replace_with(new_element)

Loading…
Cancel
Save