From 6174f627102edbfd0f89a65e91376b9f06148c96 Mon Sep 17 00:00:00 2001 From: tribta Date: Fri, 17 Nov 2017 12:56:47 -0500 Subject: [PATCH] Sample for functions --- doc/CMakeLists.txt | 6 +- doc/tools/add_signatures.py | 101 +++++++++++++ doc/tools/doxygen_scan.py | 51 +++++++ doc/tools/html_functions.py | 266 +++++++++++++++------------------ doc/tools/python_signatures.py | 58 ------- 5 files changed, 276 insertions(+), 206 deletions(-) create mode 100644 doc/tools/add_signatures.py create mode 100644 doc/tools/doxygen_scan.py delete mode 100644 doc/tools/python_signatures.py diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 741c4a093e..d79b86c36b 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -205,7 +205,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}") endforeach() - set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/index.html") + set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/modules.html") add_custom_target(doxygen_cpp COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} @@ -220,7 +220,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) if(BUILD_opencv_python2) add_custom_target(doxygen_python - COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" + COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python2 ) add_custom_target(doxygen @@ -228,7 +228,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) ) elseif(BUILD_opencv_python3) add_custom_target(doxygen_python - COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" + COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python3 ) add_custom_target(doxygen diff --git a/doc/tools/add_signatures.py b/doc/tools/add_signatures.py new file mode 100644 index 0000000000..05c029de25 --- /dev/null +++ b/doc/tools/add_signatures.py @@ -0,0 +1,101 @@ +""" +This code adds Python/Java signatures to the docs. + +TODO: Do the same thing for Java +* using javadoc/ get all the methods/classes/constants to a json file + +TODO: +* clarify when there are several C++ signatures corresponding to a single Python function. + i.e: calcHist(): + http://docs.opencv.org/3.2.0/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d +* clarify special case: + http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 +""" +from __future__ import print_function +import os +import re +import sys +import logging +import html_functions +import doxygen_scan + +loglevel=os.environ.get("LOGLEVEL", None) +if loglevel: + logging.basicConfig(level=loglevel) + + +ROOT_DIR = sys.argv[1] +PYTHON_SIGNATURES_FILE = sys.argv[2] +JAVA_PYTHON = sys.argv[3] + +ADD_JAVA = False +ADD_PYTHON = False +if JAVA_PYTHON == "python": + ADD_PYTHON = True + +import json +python_signatures = dict() +with open(PYTHON_SIGNATURES_FILE, "rt") as f: + python_signatures = json.load(f) + print("Loaded Python signatures: %d" % len(python_signatures)) + +# only name -> class +# name and ret -> constant +# name, ret, arg-> function / class method + +class Configuration(): + def __init__(self): + self.ADD_PYTHON = ADD_PYTHON + self.python_signatures = python_signatures + self.ADD_JAVA = ADD_JAVA + +config = Configuration() + +import xml.etree.ElementTree as ET +root = ET.parse(ROOT_DIR + 'opencv.tag') +files_dict = dict() + +# constants and function from opencv.tag +namespaces = root.findall("./compound[@kind='namespace']") +#print("Found {} namespaces".format(len(namespaces))) +for ns in namespaces: + ns_name = ns.find("./name").text + #print('NS: {}'.format(ns_name)) + + files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) + files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) + +# class methods from opencv.tag +classes = root.findall("./compound[@kind='class']") +#print("Found {} classes".format(len(classes))) +for c in classes: + c_name = c.find("./name").text + name = ns_name + '::' + c_name + file = c.find("./filename").text + #print('Class: {} => {}'.format(name, file)) + files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict) + +# test +for file in files_dict: + soup = html_functions.load_html_file(ROOT_DIR + file) + if file == "d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": + anchor_list = files_dict[file] + counter = 0 + anchor_tmp_list = [] + for anchor in anchor_list: + counter += 1 + # if the next anchor shares the same C++ name (= same method/function), join them together + if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname: + anchor_tmp_list.append(anchor) + continue + else: + anchor_tmp_list.append(anchor) + # check if extists a python equivalent signature + for signature in python_signatures: # signature is a key with the C++ name + if signature == anchor.cppname: # if available name in python + # they should also have the same type + soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup) + #print(signature) + # reset anchor temporary list + anchor_tmp_list[:] = [] + html_functions.update_html(ROOT_DIR + file, soup) diff --git a/doc/tools/doxygen_scan.py b/doc/tools/doxygen_scan.py new file mode 100644 index 0000000000..4567723302 --- /dev/null +++ b/doc/tools/doxygen_scan.py @@ -0,0 +1,51 @@ +class Anchor(object): + anchor = "" + type = "" + cppname = "" + + def __init__(self, anchor, type, cppname): + self.anchor = anchor + self.type = type + self.cppname = cppname + +def add_to_file(files_dict, file, anchor): + if file in files_dict: + # if that file already exists as a key in the dictionary + files_dict[file].append(anchor) + else: + files_dict[file] = [anchor] + return files_dict + + +def scan_namespace_constants(ns, ns_name, files_dict): + constants = ns.findall("./member[@kind='enumvalue']") + for c in constants: + c_name = c.find("./name").text + name = ns_name + '::' + c_name + file = c.find("./anchorfile").text + anchor = c.find("./anchor").text + #print(' CONST: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name)) + return files_dict + +def scan_namespace_functions(ns, ns_name, files_dict): + functions = ns.findall("./member[@kind='function']") + for f in functions: + f_name = f.find("./name").text + name = ns_name + '::' + f_name + file = f.find("./anchorfile").text + anchor = f.find("./anchor").text + #print(' FN: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name)) + return files_dict + +def scan_class_methods(c, c_name, files_dict): + methods = c.findall("./member[@kind='function']") + for m in methods: + m_name = m.find("./name").text + name = c_name + '::' + m_name + file = m.find("./anchorfile").text + anchor = m.find("./anchor").text + #print(' Method: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name)) + return files_dict diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index 215a0581d4..ca3e096cde 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -1,7 +1,6 @@ from __future__ import print_function import logging import os -import codecs from pprint import pprint try: @@ -12,180 +11,157 @@ except ImportError: 'Install BeautifulSoup (bs4) for adding' ' Python & Java signatures documentation') - -def is_not_module_link(tmp_link): - """ Checks if a link belongs to a c++ method """ - if tmp_link is None: - return True - if "group" not in tmp_link: - return True - if "#" in tmp_link: - return True - return False - - -def get_links_list(tmp_soup, filter_links): - """ Get a list of links from a soup """ - tmp_href_list = [] - for tmp_link in tmp_soup.findAll('a'): - tmp_href = tmp_link.get('href') - if filter_links: - if is_not_module_link(tmp_href): - continue - tmp_href_list.append(tmp_href) - return tmp_href_list - - def load_html_file(file_dir): """ Uses BeautifulSoup to load an html """ with open(file_dir) as fp: - tmp_soup = BeautifulSoup(fp, 'html.parser') - return tmp_soup - + soup = BeautifulSoup(fp, 'html.parser') + return soup -def add_item(tmp_soup, new_row, is_parameter, text): +def add_item(soup, new_row, is_parameter, text): """ Adds a new html tag for the table with the signature """ - new_item = tmp_soup.new_tag('td') + new_item = soup.new_tag('td') if is_parameter: - new_item = tmp_soup.new_tag('td', **{'class': 'paramname'}) + new_item = soup.new_tag('td', **{'class': 'paramname'}) new_item.append(text) new_row.append(new_item) - return new_row + return new_row, soup - -def get_text_between_substrings(sig, begin_char, end_char): - return sig.partition(begin_char)[-1].rpartition(end_char)[0] - - -def add_signature_to_table(tmp_soup, new_row, signature, function_name, language, ident): +def add_signature_to_table(soup, tmp_row, signature, language): """ Add a signature to an html table""" - if ident: - new_item = tmp_soup.new_tag('td', style="padding-left: 0.5cm;") - else: - new_item = tmp_soup.new_tag('td') + new_item = soup.new_tag('td', style="padding-left: 0.5cm;") if str(signature.get('ret', None)) != "None": new_item.append(signature.get('ret') + ' =') - new_row.append(new_item) - - if "Python" in language: - pass # function_name = "cv2." + function_name - elif "Java" in language: - # get word before function_name (= output) - str_before_bracket = signature.split('(', 1)[0] - list_of_words = str_before_bracket.split() - output = list_of_words[len(list_of_words) - 2] - new_item.append(output + " ") - new_row.append(new_item) + tmp_row.append(new_item) - new_row = add_item(tmp_soup, new_row, False, signature.get('name', function_name) + '(') - new_row = add_item(tmp_soup, new_row, True, signature['arg']) - new_row = add_item(tmp_soup, new_row, False, ')') - return new_row + tmp_row, soup = add_item(soup, tmp_row, False, "cv2." + signature.get('name', None) + '(') + tmp_row, soup = add_item(soup, tmp_row, True, signature['arg']) + tmp_row, soup = add_item(soup, tmp_row, False, ')') + return tmp_row, soup -def new_line(tmp_soup, tmp_table, new_row): +def new_line(soup, tmp_table, new_row): """ Adds a new line to the html table """ tmp_table.append(new_row) - new_row = tmp_soup.new_tag('tr') - return new_row + new_row = soup.new_tag('tr') + return new_row, soup -def add_bolded(tmp_soup, new_row, text): +def add_bolded(soup, new_row, text): """ Adds bolded text to the table """ - new_item = tmp_soup.new_tag('th', style="text-align:left") + new_item = soup.new_tag('th', style="text-align:left") new_item.append(text) new_row.append(new_item) - return new_row + return new_row, soup -def create_description(tmp_soup, language, signatures, function_name): +def create_description(soup, language, signatures): """ Insert the new Python / Java table after the current html c++ table """ assert signatures - tmp_table = tmp_soup.new_tag('table') - new_row = tmp_soup.new_tag('tr') - new_row = add_bolded(tmp_soup, new_row, language) - ident = False - - new_row = new_line(tmp_soup, tmp_table, new_row) - ident = True - + tmp_table = soup.new_tag('table') + new_row = soup.new_tag('tr') + new_row, soup = add_bolded(soup, new_row, language) + new_row, soup = new_line(soup, tmp_table, new_row) for s in signatures: - new_row = new_line(tmp_soup, tmp_table, new_row) - new_row = add_signature_to_table(tmp_soup, new_row, s, function_name, language, ident) - new_row = new_line(tmp_soup, tmp_table, new_row) - - return tmp_table - - -def add_signatures(tmp_soup, tmp_dir, module_name, config): - """ Add signatures to the current soup and rewrite the html file""" - - logging.debug(tmp_dir) - sign_counter = 0 - python_sign_counter = 0 - java_sign_counter = 0 - - if config.ADD_JAVA: - functions_file = "java_doc_txts/" + module_name + "/functions.txt" - if os.path.exists(functions_file): - with open(functions_file, 'r') as f: - java_signatures = f.read().split("\n") + new_row, soup = new_line(soup, tmp_table, new_row) + new_row, soup = add_signature_to_table(soup, new_row, s, language) + new_row, soup = new_line(soup, tmp_table, new_row) + return tmp_table, soup + + +def get_anchor_list(anchor, soup): + a_list = [] + # go through all the links + for a in soup.find_all('a', href=True): + # find links with the same anchor + last_part_of_link = a['href'].rsplit('#', 1)[-1] + if last_part_of_link == anchor: + a_list.append(a) + return a_list + +def append_python_signatures_to_table(soup, signatures, table): + description, soup = create_description(soup, "Python:", signatures) + description['class'] = 'python_language' + old = table.next_sibling + if old.name != 'table': + old = None + elif not 'python_language' in old.get('class', []): + old = None + # if already existed replace with the new + if old is None: + table.insert_after(description) + else: + old.replace_with(description) + table.insert_after(description) + return soup + +def get_heading_text(a): + str = "" + element = a.parent.parent + if element is not None: + if element.has_attr('class'): + tmp_class = element["class"][0] + if "memitem:" in tmp_class and "python" not in tmp_class: + str = element.parent.find("tr").text + return str + +def append_python_signatures_to_heading(soup, signatures, element, href): + for s in signatures: + attrs = {'class': 'memitem:python'} + new_tr = soup.new_tag('tr', **attrs) + + attrs_left = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} + new_td_left = soup.new_tag('td', **attrs_left) + new_td_left.append(str(s.get('ret', None))) + new_tr.append(new_td_left) + + attrs_right = {'class': 'memItemRight', 'valign': 'bottom'} + new_td_right = soup.new_tag('td', **attrs_right) + attrs_a = {'class': 'el', 'href': href} + new_a = soup.new_tag('a', **attrs_a) + new_a.append('cv2.' + str(s.get('name', None))) + new_td_right.append(new_a) + new_td_right.append("(" + s['arg'] +")") + + new_tr.append(new_td_right) + + old = element.next_sibling + if old is not None: + if old.name != 'tr': + old = None + elif not 'memitem:python' in old.get('class', []): + old = None + # if already existed replace with the new + if old is None: + element.insert_after(new_tr) else: - config.ADD_JAVA = False # This C++ module (module_name) may not exist in Java - - # the HTML tag & class being used to find functions - for function in tmp_soup.findAll("h2", {"class": "memtitle"}): - function_name = None - for c in function.contents: - if isinstance(c, bs4.element.NavigableString): - fn = str(c).encode("ascii","ignore").decode().strip() - if not fn.endswith('()'): # all functions have () in it's name - # enums, structures, etc - continue - function_name = fn[:-2] - - if not function_name: - continue - - sign_counter += 1 - - cpp_table = function.findNext('table') - - if config.ADD_PYTHON: - signatures = config.python_signatures.get("cv::" + str(function_name), None) - if signatures: - print(function_name) - - description = create_description(tmp_soup, "Python:", signatures, function_name) - description['class'] = 'python_language' - old = cpp_table.next_sibling - if old.name != 'table': - old = None - elif not 'python_language' in old.get('class', []): - old = None - if old is None: - cpp_table.insert_after(description) - else: - old.replace_with(description) - python_sign_counter += 1 - - if config.ADD_JAVA: - for signature in java_signatures: - if function_name in signature: - create_description(cpp_table, tmp_soup, "Java:", signature, function_name) - java_sign_counter += 1 - break - - tmp_str = str(tmp_soup) + old.replace_with(new_tr) + return soup + +def append_python_signature(function_variants, anchor_list, soup): + type = anchor_list[0].type + if type == "fn": + if len(anchor_list) == 1: + tmp_anchor = anchor_list[0].anchor + a_list = get_anchor_list(tmp_anchor, soup) + for a in a_list: + if a['href'] == "#" + tmp_anchor: + # Function Documentation (tables) + table = a.findNext('table') + if table is not None: + soup = append_python_signatures_to_table(soup, function_variants, table) + continue + str = get_heading_text(a) + if "Functions" in str: + soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href']) + print("One more") + return soup + +def update_html(file, soup): + tmp_str = str(soup) if os.name == 'nt': # if Windows - with open(tmp_dir, "wb") as tmp_file: + with open(file, "wb") as tmp_file: tmp_file.write(tmp_str.encode("ascii","ignore")) else: - with open(tmp_dir, "w") as tmp_file: + with open(file, "w") as tmp_file: tmp_file.write(tmp_str) - - logging.debug("Added [" + str(python_sign_counter) + \ - "/" + str(sign_counter) + "] Python signatures") - logging.debug("Added [" + str(java_sign_counter) + \ - "/" + str(sign_counter) + "] Java signatures") diff --git a/doc/tools/python_signatures.py b/doc/tools/python_signatures.py deleted file mode 100644 index 6908de278a..0000000000 --- a/doc/tools/python_signatures.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -This code adds Python signatures to the docs. - -TODO: -* clarify when there are several C++ signatures corresponding to a single Python function. - i.e: calcHist(): - http://docs.opencv.org/3.2.0/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d -* clarify special case: - http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 -""" -from __future__ import print_function -import os -import re -import sys -import logging - -loglevel=os.environ.get("LOGLEVEL", None) -if loglevel: - logging.basicConfig(level=loglevel) - -ADD_JAVA = False -ADD_PYTHON = True -ROOT_DIR = sys.argv[1] -PYTHON_SIGNATURES_FILE = sys.argv[2] - -import json -python_signatures = dict() -with open(PYTHON_SIGNATURES_FILE, "rt") as f: - python_signatures = json.load(f) - print("Loaded Python signatures: %d" % len(python_signatures)) - -class Configuration(): - def __init__(self): - self.ADD_PYTHON = ADD_PYTHON - self.python_signatures = python_signatures - self.ADD_JAVA = ADD_JAVA - -config = Configuration() - - -import html_functions - -soup = html_functions.load_html_file(ROOT_DIR + "index.html") -href_list = html_functions.get_links_list(soup, True) - -for link in href_list: - # add python signatures to the module - soup = html_functions.load_html_file(ROOT_DIR + link) - sub_href_list = html_functions.get_links_list(soup, True) - module_name = html_functions.get_text_between_substrings(link, "group__", ".html") - html_functions.add_signatures(soup, ROOT_DIR + link, module_name, config) - - # add python signatures to the sub-modules - link = re.sub(r"group__.+html", "", link) - for sub_link in sub_href_list: - tmp_dir = ROOT_DIR + link + sub_link - soup = html_functions.load_html_file(tmp_dir) - html_functions.add_signatures(soup, tmp_dir, module_name, config)