Improved rst_parser and javadoc generation scripts

modules/java/gen_javadoc.py

@@ -1,11 +1,22 @@
 import os, sys, re, string, glob
-
+allmodules = ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]
-javadoc_marker = "//javadoc:"
+verbose = False
-
+show_warnings = True
-def parceJavadocMarker(line):
+show_errors = True
-    assert line.lstrip().startswith(javadoc_marker)
+
-    offset = line[:line.find(javadoc_marker)]
+class JavadocGenerator(object):
-    line = line.strip()[len(javadoc_marker):]
+    def __init__(self, definitions = {}, javadoc_marker = "//javadoc:"):
+        self.definitions = definitions
+        self.javadoc_marker = javadoc_marker
+        self.markers_processed = 0
+        self.markers_documented = 0
+        self.params_documented = 0
+        self.params_undocumented = 0
+
+    def parceJavadocMarker(self, line):
+        assert line.lstrip().startswith(self.javadoc_marker)
+        offset = line[:line.find(self.javadoc_marker)]
+        line = line.strip()[len(self.javadoc_marker):]
         args_start = line.rfind("(")
         args_end = line.rfind(")")
         assert args_start * args_end > 0
@@ -14,19 +25,29 @@ def parceJavadocMarker(line):
             return (line[:args_start].strip(), offset,  filter(None, list(arg.strip() for arg in line[args_start+1:args_end].split(","))))
         return (line, offset, [])
 
-def document(infile, outfile, decls):
+    def document(self, infile, outfile):
         inf = open(infile, "rt")
         outf = open(outfile, "wt")
+        module = os.path.splitext(os.path.basename(infile))[0]
+        if module not in allmodules:
+            module = "unknown"
         try:
             for l in inf.readlines():
-            if l.lstrip().startswith(javadoc_marker):
+                if l.lstrip().startswith(self.javadoc_marker):
-                marker = parceJavadocMarker(l)
+                    marker = self.parceJavadocMarker(l)
-                decl = decls.get(marker[0],None)
+                    self.markers_processed += 1
+                    decl = self.definitions.get(marker[0],None)
                     if decl:
-                    for line in makeJavadoc(decl, decls, marker[2]).split("\n"):
+                        javadoc = self.makeJavadoc(decl, marker[2])
+                        if verbose:
+                            print
+                            print "Javadoc for \"%s\" File: %s (line %s)" % (decl["name"], decl["file"], decl["line"])
+                            print javadoc
+                        for line in javadoc.split("\n"):
                             outf.write(marker[1] + line + "\n")
-                else:
+                        self.markers_documented += 1
-                    print "Error: could not find documentation for %s" % l.lstrip()[len(javadoc_marker):-1]
+                    elif show_errors:
+                        print >> sys.stderr, "gen_javadoc error: could not find documentation for %s (module: %s)" % (l.lstrip()[len(self.javadoc_marker):-1].strip(), module)
                 else:
                     outf.write(l.replace("\t", "    ").rstrip()+"\n")
         except:
@@ -38,7 +59,7 @@ def document(infile, outfile, decls):
             inf.close()
             outf.close()
 
-def ReformatForJavadoc(s):
+    def ReformatForJavadoc(self, s):
         out = ""
         for term in s.split("\n"):
             if term.startswith("*") or term.startswith("#."):
@@ -67,7 +88,7 @@ def ReformatForJavadoc(s):
                     pos_end = min(pos_start + 77, len(term)-1)
         return out
 
-def getJavaName(decl):
+    def getJavaName(self, decl):
         name = "org.opencv."
         name += decl["module"]
         if "class" in decl:
@@ -78,7 +99,7 @@ def getJavaName(decl):
             name += "." + decl["method"]
         return name
 
-def getDocURL(decl):
+    def getDocURL(self, decl):
         url = "http://opencv.itseez.com/modules/"
         url += decl["module"]
         url += "/doc/"
@@ -86,7 +107,7 @@ def getDocURL(decl):
         url += "#" + decl["name"].replace("::","-").replace("()","").replace("=","").strip().rstrip("_").replace(" ","-").replace("_","-").lower()
         return url
 
-def makeJavadoc(decl, decls, args = None):
+    def makeJavadoc(self, decl, args = None):
         doc = ""
         prefix = "/**\n"
 
@@ -101,16 +122,17 @@ def makeJavadoc(decl, decls, args = None):
 
         # brief goes first
         if "brief" in decl:
-        doc += prefix + ReformatForJavadoc(decl["brief"])
+            doc += prefix + self.ReformatForJavadoc(decl["brief"])
             prefix = " *\n"
         elif "long" not in decl:
-        print "Warning: no description for " + decl_type + " \"%s\" File: %s (line %s)" % (func["name"], func["file"], func["line"])
+            if show_warnings:
-        doc += prefix + ReformatForJavadoc("This " + decl_type + " is undocumented")
+                print >> sys.stderr, "gen_javadoc warning: no description for " + decl_type + " \"%s\" File: %s (line %s)" % (func["name"], func["file"], func["line"])
+            doc += prefix + self.ReformatForJavadoc("This " + decl_type + " is undocumented")
             prefix = " *\n"
     
         # long goes after brief
         if "long" in decl:
-        doc += prefix  + ReformatForJavadoc(decl["long"])
+            doc += prefix  + self.ReformatForJavadoc(decl["long"])
             prefix = " *\n"
 
         # @param tags
@@ -120,21 +142,25 @@ def makeJavadoc(decl, decls, args = None):
                 arg_doc = documented_params.get(arg, None)
                 if not arg_doc:
                     arg_doc = "a " + arg
-                print "Warning: parameter \"%s\" of \"%s\" is undocumented. File: %s (line %s)" % (arg, decl["name"], decl["file"], decl["line"])
+                    if show_warnings:
-            doc += prefix + ReformatForJavadoc("@param " + arg + " " + arg_doc)
+                        print >> sys.stderr, "gen_javadoc warning: parameter \"%s\" of \"%s\" is undocumented. File: %s (line %s)" % (arg, decl["name"], decl["file"], decl["line"])
+                    self.params_undocumented += 1
+                else:
+                    self.params_documented += 1
+                doc += prefix + self.ReformatForJavadoc("@param " + arg + " " + arg_doc)
                 prefix = ""
             prefix = " *\n"
 
         # @see tags
         # always link to documentation
-    doc += prefix + " * @see <a href=\"" + getDocURL(decl) + "\">" + getJavaName(decl) + "</a>\n"
+        doc += prefix + " * @see <a href=\"" + self.getDocURL(decl) + "\">" + self.getJavaName(decl) + "</a>\n"
         prefix = ""
         # other links
         if "seealso" in decl:
             for see in decl["seealso"]:
-            seedecl = decls.get(see,None)
+                seedecl = self.definitions.get(see,None)
                 if seedecl:
-                doc += prefix + " * @see " + getJavaName(seedecl) + "\n"
+                    doc += prefix + " * @see " + self.getJavaName(seedecl) + "\n"
                 else:
                     doc += prefix + " * @see " + see.replace("::",".") + "\n"
         prefix = " *\n"
@@ -143,6 +169,18 @@ def makeJavadoc(decl, decls, args = None):
 
         return (doc + " */").replace("::",".")
 
+    def printSummary(self):
+        print
+        print "Javadoc Generator Summary:"
+        print "  Total markers:        %s" % self.markers_processed
+        print "  Undocumented markers: %s" % (self.markers_processed - self.markers_documented)
+        print "  Generated comments:   %s" % self.markers_documented
+
+        print
+        print "  Documented params:    %s" % self.params_documented
+        print "  Undocumented params:  %s" % self.params_undocumented
+        print
+
 if __name__ == "__main__":
     if len(sys.argv) < 2:
         print "Usage:\n", os.path.basename(sys.argv[0]), " <input dir1> [<input dir2> [...]]"
@@ -156,16 +194,19 @@ if __name__ == "__main__":
     import hdr_parser
     import rst_parser
 
-    parser = rst_parser.RstParser(hdr_parser.CppHeaderParser())
-    
     print "Parsing documentation..."
-    for m in ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]:
+    parser = rst_parser.RstParser(hdr_parser.CppHeaderParser())
+    for m in allmodules:
         parser.parse(m, os.path.join(selfpath, "../" + m))
         
     parser.printSummary()
 
+    print "Generating javadoc comments..."
+    generator = JavadocGenerator(parser.definitions)
     for i in range(1, len(sys.argv)):
         folder = os.path.abspath(sys.argv[i])
         for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]:
             outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java"))
-            document(jfile, outfile, parser.definitions)
+            generator.document(jfile, outfile)
+
+    generator.printSummary()

modules/java/rst_parser.py

@@ -1,4 +1,5 @@
 import os, sys, re, string, glob
+allmodules = ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]
 verbose = False
 show_warnings = True
 show_errors = True
@@ -564,6 +565,7 @@ class RstParser(object):
         print "  structs documented:           %s" % structs
         for lang in sorted(stat.items()):
             print "  %7s functions documented: %s" % lang
+        print
 
 def mathReplace2(match):
     m = mathReplace(match)
@@ -670,7 +672,7 @@ if __name__ == "__main__":
     parser = RstParser(hdr_parser.CppHeaderParser())
     
     if module == "all":
-        for m in ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]:
+        for m in allmodules:
             parser.parse(m, os.path.join(rst_parser_dir, "../" + m))
     else:
         parser.parse(module, os.path.join(rst_parser_dir, "../" + module))