Merge pull request #267 from s-koch/python_scripts

Added automatic binding generator script

Former-commit-id: b6378ac2d98a93fab668ded0750ee1562f643e57

python/scripts/basic_function.mako

@@ -0,0 +1,46 @@
+% for enum in enums:
+py::enum_<\
+% for n in enum['namespaces']:
+${n}::\
+% endfor
+${enum['name']}>(m, "${enum['name']}")
+% for c in enum['constants']:
+    .value("${c}", \
+% for n in enum['namespaces']:
+${n}::\
+% endfor
+${c})
+% endfor
+    .export_values();
+% endfor
+
+
+% for func in functions:
+m.def("${func['name']}", []
+(
+  % for p in func['parameters'][:-1]:
+  ${p['type']} ${p['name']},
+  % endfor
+  ${func['parameters'][-1]['type']} ${func['parameters'][-1]['name']}
+)
+{
+  return \
+% for n in func['namespaces']:
+${n}::\
+% endfor
+${func['name']}(\
+% for p in func['parameters'][:-1]:
+${p['name']}, \
+% endfor
+${func['parameters'][-1]['name']});
+}, __doc_\
+% for n in func['namespaces']:
+${n}_\
+% endfor
+${func['name']},
+% for p in func['parameters'][:-1]:
+py::arg("${p['name']}"), \
+% endfor
+py::arg("${func['parameters'][-1]['name']}"));
+
+% endfor

python/scripts/generate_bindings.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+#
+#  Syntax: generate_docstrings.py <path_to_c++_header_files> <path_to_python_files>
+#
+#  Extract documentation from C++ header files to use it in libiglPython bindings
+#
+
+import os, sys, glob
+import pickle
+
+import shutil
+from joblib import Parallel, delayed
+from multiprocessing import cpu_count
+from mako.template import Template
+from parser import parse
+
+
+# http://stackoverflow.com/questions/3207219/how-to-list-all-files-of-a-directory-in-python
+def get_filepaths(directory):
+    """
+    This function will generate the file names in a directory
+    tree by walking the tree either top-down or bottom-up. For each
+    directory in the tree rooted at directory top (including top itself),
+    it yields a 3-tuple (dirpath, dirnames, filenames).
+    """
+    file_paths = []  # List which will store all of the full filepaths.
+
+    # Walk the tree.
+    for root, directories, files in os.walk(directory):
+        for filename in files:
+            # Join the two strings in order to form the full filepath.
+            filepath = os.path.join(root, filename)
+            file_paths.append(filepath)  # Add it to the list.
+
+    return file_paths  # Self-explanatory.
+
+
+def get_name_from_path(path, basepath, prefix, postfix):
+    f_clean = path[len(basepath):]
+    f_clean = f_clean.replace(basepath, "")
+    f_clean = f_clean.replace(postfix, "")
+    f_clean = f_clean.replace(prefix, "")
+    f_clean = f_clean.replace("/", "_")
+    f_clean = f_clean.replace("\\", "_")
+    f_clean = f_clean.replace(" ", "_")
+    f_clean = f_clean.replace(".", "_")
+    return f_clean
+
+
+def map_parameter_types(name, cpp_type, parsed_types, errors, enum_types):
+    # TODO Replace with proper regex matching and derive types from templates, comment parsing, names in cpp files
+    # CAUTION: This is work in progress mapping code to get a grip of the problem
+    # Types to map
+    #    const int dim -> const int& dim ?
+    result = []
+
+    if cpp_type.startswith("const"):
+        result.append("const ")
+        cpp_type = cpp_type[6:]  # Strip const part
+
+    # Handle special types
+    skip_parsing = False
+    if cpp_type.startswith("MatY"):
+        result.append("Eigen::SparseMatrix<double>&")
+        skip_parsing = True
+    if cpp_type == "std::vector<std::vector<Scalar> > &":
+        result.append("std::vector<std::vector<double> > &")
+        skip_parsing = True
+    if cpp_type == "std::vector<std::vector<Index> > &":
+        result.append("std::vector<std::vector<int> > &")
+        skip_parsing = True
+    for constant in enum_types:
+        if cpp_type.endswith(constant):
+            result.append(cpp_type)
+            skip_parsing = True
+
+    if len(parsed_types) == 0:
+        errors.append("Empty typechain: %s" % cpp_type)
+        if cpp_type == "int" or cpp_type == "bool":
+            return cpp_type, True
+        else:
+            return cpp_type, False
+
+    # print(parsed_types, cpp_type)
+    if not skip_parsing:
+        for i, t in enumerate(parsed_types):
+
+            if t == "Eigen":
+                result.append("Eigen::")
+                continue
+            if t == "std":
+                result.append("std::")
+                continue
+
+            if t == "PlainObjectBase" or t == "MatrixBase":
+                if name == "F":
+                    result.append("MatrixXi&")
+                elif name == "V":
+                    result.append("MatrixXd&")
+                else:
+                    result.append("MatrixXd&")
+                break
+            if t == "MatrixXi":
+                result.append("MatrixXi&")
+                break
+            if t == "MatrixXd" or t == "VectorXd":
+                result.append("MatrixXd&")
+                break
+            if t == "SparseMatrix" and len(parsed_types) >= i + 2 and (
+                    parsed_types[i + 1] == "Scalar" or parsed_types[i + 1] == "T"):
+                result.append("SparseMatrix<double>&")
+                break
+            if t == "SparseVector" and len(parsed_types) >= i + 2 and (parsed_types[i + 1] == "Scalar" or parsed_types[
+                    i + 1] == "T"):
+                result.append("SparseMatrix<double>&")
+                break
+
+            if t == "bool" or t == "int" or t == "double" or t == "unsigned" or t == "string":
+                if cpp_type.endswith("&"):
+                    result.append(t + " &")
+                else:
+                    result.append(t)
+                break
+
+            else:
+                errors.append("Unknown typechain: %s" % cpp_type)
+                return cpp_type, False
+
+
+    return "".join(result), True
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 2:
+        print('Syntax: %s <path_to_c++_files>' % sys.argv[0])
+        exit(-1)
+
+    errors = {"missing": [], "empty": [], "others": [], "incorrect": [], "render": [], "various": []}
+    files = {"complete": [], "partial": [], "errors": [], "others": [], "empty": []}
+
+    # List all files in the given folder and subfolders
+    cpp_base_path = sys.argv[1]
+    cpp_file_paths = get_filepaths(cpp_base_path)
+
+    # Add all the .h filepaths to a dict
+    print("Collecting cpp files for parsing...")
+    mapping = {}
+    cppmapping = {}
+    for f in cpp_file_paths:
+        if f.endswith(".h"):
+            name = get_name_from_path(f, cpp_base_path, "", ".h")
+            mapping[name] = f
+
+        if f.endswith(".cpp"):
+            name = get_name_from_path(f, cpp_base_path, "", ".cpp")
+            cppmapping[name] = f
+
+    # Add all python binding files to a list
+    implemented_names = list(mapping.keys())  # ["point_mesh_squared_distance"]
+    implemented_names.sort()
+    single_postfix = ""
+    single_prefix = ""
+
+    # Create a list of all cpp header files
+    files_to_parse = []
+    cppfiles_to_parse = []
+    for n in implemented_names:
+        files_to_parse.append(mapping[n])
+
+        if n not in cppmapping:
+            errors["missing"].append("No cpp source file for function %s found." % n)
+        else:
+            cppfiles_to_parse.append(cppmapping[n])
+
+    # Parse c++ header files
+    print("Parsing header files...")
+    load_headers = False
+    if load_headers:
+        with open("headers.dat", 'rb') as fs:
+            dicts = pickle.load(fs)
+    else:
+        job_count = cpu_count()
+        dicts = Parallel(n_jobs=job_count)(delayed(parse)(path) for path in files_to_parse)
+
+    if not load_headers:
+        print("Saving parsed header files...")
+        with open("headers.dat", 'wb') as fs:
+            pickle.dump(dicts, fs)
+
+    # Not yet needed, as explicit template parsing does not seem to be supported in clang
+    # Parse c++ source files
+    # cppdicts = Parallel(n_jobs=job_count)(delayed(parse)(path) for path in cppfiles_to_parse)
+
+    # Change directory to become independent of execution directory
+    print("Generating directory tree for binding files...")
+    path = os.path.dirname(__file__)
+    if path != "":
+        os.chdir(path)
+    shutil.rmtree("generated")
+    os.makedirs("generated/complete")
+    os.mkdir("generated/partial")
+
+    print("Generating and writing binding files...")
+    for idx, n in enumerate(implemented_names):
+        d = dicts[idx]
+        contained_elements = sum(map(lambda x: len(x), d.values()))
+
+        # Skip files that don't contain functions/enums/classes
+        if contained_elements == 0:
+            errors["empty"].append("Function %s contains no parseable content in cpp header. Something might be wrong." % n)
+            files["empty"].append(n)
+            continue
+
+        # Add functions with classes to others
+        if len(d["classes"]) != 0 or len(d["structs"]) != 0:
+            errors["others"].append("Function %s contains classes/structs in cpp header. Skipping" % n)
+            files["others"].append(n)
+            continue
+
+        # Work on files that contain only functions/enums and namespaces
+        if len(d["functions"]) + len(d["namespaces"]) + len(d["enums"]) == contained_elements:
+            correct_functions = []
+            incorrect_functions = []
+
+            # Collect enums to generate binding files
+            enums = []
+            enum_types = []
+            for e in d["enums"]:
+                enums.append({"name": e.name, "namespaces": d["namespaces"], "constants": e.constants})
+                enum_types.append(e.name)
+
+            # Collect functions to generate binding files
+            for f in d["functions"]:
+                parameters = []
+                correct_function = True
+                f_errors = []
+                for p in f.parameters:
+                    typ, correct = map_parameter_types(p[0], p[1], p[2], f_errors, enum_types)
+                    correct_function &= correct
+                    parameters.append({"name": p[0], "type": typ})
+
+                if correct_function and len(parameters) > 0: #TODO add constants like EPS
+                    correct_functions.append({"parameters": parameters, "namespaces": d["namespaces"], "name": f.name})
+                elif len(parameters) > 0:
+                    incorrect_functions.append({"parameters": parameters, "namespaces": d["namespaces"], "name": f.name})
+                    errors["incorrect"].append("Incorrect function in %s: %s, %s\n" % (n, f.name, ",".join(f_errors)))
+                else:
+                    errors["various"].append("Function without pars in %s: %s, %s\n" % (n, f.name, ","
+                                                                                                     "".join(f_errors)))
+
+            # Write binding files
+            try:
+                tpl = Template(filename='basic_function.mako')
+                rendered = tpl.render(functions=correct_functions, enums=enums)
+                tpl1 = Template(filename='basic_function.mako')
+                rendered1 = tpl.render(functions=incorrect_functions, enums=enums)
+                path = "generated/"
+                if len(incorrect_functions) == 0 and (len(correct_functions) != 0 or len(enums) != 0):
+                    path += "complete/"
+                    with open(path + single_prefix + "py_" + n + ".cpp", 'w') as fs:
+                        fs.write(rendered)
+                    files["complete"].append(n)
+                else:
+                    path += "partial/"
+                    with open(path + single_prefix + "py_" + n + ".cpp", 'w') as fs:
+                        fs.write("// COMPLETE BINDINGS ========================\n")
+                        fs.write(rendered)
+                        fs.write("\n\n\n\n// INCOMPLETE BINDINGS ========================\n")
+                        fs.write(rendered1)
+
+                    if len(correct_functions) != 0:
+                        files["partial"].append(n)
+                    else:
+                        files["errors"].append(n)
+
+            except Exception as e:
+                files["errors"].append(n)
+                errors["render"].append("Template rendering failed:" + n + " " + str(correct_functions) + ", incorrect "
+                                                                                                "functions are " + str(
+                    incorrect_functions) + str(e) + "\n")
+
+    print("Writing error and overview files...")
+    with open("errors.txt" + single_postfix, 'w') as fs:
+        l = list(errors.keys())
+        l.sort()
+        for k in l:
+            fs.write("%s: %i \n" %(k, len(errors[k])))
+            fs.writelines("\n".join(errors[k]))
+            fs.write("\n\n\n")
+
+    with open("files.txt" + single_postfix, 'w') as fs:
+        l = list(files.keys())
+        l.sort()
+        for k in l:
+            fs.write("%s: %i \n" %(k, len(files[k])))
+            fs.writelines("\n".join(files[k]))
+            fs.write("\n\n\n")

python/scripts/generate_docstrings.py

@@ -87,16 +87,16 @@ if __name__ == '__main__':
     cpplines = []
 
     for idx, n in enumerate(implemented_names):
-        dict = dicts[idx]
-        contained_elements = sum(map(lambda x: len(x), dict.values()))
+        d = dicts[idx]
+        contained_elements = sum(map(lambda x: len(x), d.values()))
         # Check for files that don't contain functions/enums/classes
         if contained_elements == 0:
             print("Function %s contains no parseable content in cpp header. Something might be wrong." % n)
             continue
         else:
             names = []
-            namespaces = "_".join(dict["namespaces"])  # Assumption that all entities lie in deepest namespace
-            for f in dict["functions"]:
+            namespaces = "_".join(d["namespaces"])  # Assumption that all entities lie in deepest namespace
+            for f in d["functions"]:
                 h_string = "extern const char *__doc_" + namespaces + "_" + f.name + ";\n"
                 docu_string = "See " + f.name + " for the documentation."
                 if f.documentation != "":

python/scripts/parser.py

@@ -3,10 +3,16 @@ import os
 from threading import Thread
 
 import clang.cindex
+import ccsyspath
 import itertools
 from mako.template import Template
 
 
+
+
+
+
+
 def get_annotations(node):
     return [c.displayname for c in node.get_children()
             if c.kind == clang.cindex.CursorKind.ANNOTATE_ATTR]
@@ -17,33 +23,33 @@ class Function(object):
         self.name = cursor.spelling
         self.annotations = get_annotations(cursor)
         self.access = cursor.access_specifier
-        #        template_pars = [c.extent for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER]
-        #        parameter_dec = [c for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.PARM_DECL]
-        #        print(parameter_dec, template_pars)
-        #        print(cursor.get_num_template_arguments(), cursor.get_template_argument_type(0), cursor.get_template_argument_value(0), template_pars, parameter_dec)
-        self.parameters = []
-        self.parnames = []
+#        template_pars = [c.extent for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER]
+        parameter_dec = [c for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.PARM_DECL]
+        
+        parameters = []
+        for p in parameter_dec:
+            children = []
+            for c in p.get_children():
+#                print(c.spelling)
+                children.append(c.spelling)
+            parameters.append((p.spelling, p.type.spelling, children))
+
+        self.parameters = parameters
         self.documentation = cursor.raw_comment
 
 
 class Enum(object):
     def __init__(self, cursor):
         self.name = cursor.spelling
-        self.annotations = get_annotations(cursor)
-        self.access = cursor.access_specifier
-        #        template_pars = [c.extent for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER]
-        #        parameter_dec = [c for c in cursor.get_children() if c.kind == clang.cindex.CursorKind.PARM_DECL]
-        #        print(parameter_dec, template_pars)
-        #        print(cursor.get_num_template_arguments(), cursor.get_template_argument_type(0), cursor.get_template_argument_value(0), template_pars, parameter_dec)
-        self.parameters = []
-        self.parnames = []
+        self.constants = [c.spelling for c in cursor.get_children() if c.kind ==
+                       clang.cindex.CursorKind.ENUM_CONSTANT_DECL]
         self.documentation = cursor.raw_comment
 
-# class Class(object):
-#    def __init__(self, cursor):
-#        self.name = cursor.spelling
+class Class(object):
+    def __init__(self, cursor):
+        self.name = cursor.spelling
 #        self.functions = []
-#        self.annotations = get_annotations(cursor)
+        self.annotations = get_annotations(cursor)
 
 #        for c in cursor.get_children():
 #            if (c.kind == clang.cindex.CursorKind.CXX_METHOD and
@@ -51,19 +57,16 @@ class Enum(object):
 #                f = Function(c)
 #                self.functions.append(f)
 
-def find_namespace_node(c):
-    if (c.kind == clang.cindex.CursorKind.NAMESPACE and c.spelling == "igl"):
-        return c
-    else:
-        for child_node in c.get_children():
-            return find_namespace_node(child_node)
 
 
 def traverse(c, path, objects):
     if c.location.file and not c.location.file.name.endswith(path):
         return
 
-    # print(c.kind, c.spelling)
+    if c.spelling == "PARULA_COLOR_MAP": # Fix to prevent python stack overflow from infinite recursion
+        return
+
+#    print(c.kind, c.spelling)
 
 
     if c.kind == clang.cindex.CursorKind.TRANSLATION_UNIT or c.kind == clang.cindex.CursorKind.UNEXPOSED_DECL:
@@ -76,7 +79,7 @@ def traverse(c, path, objects):
         pass
 
     elif c.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE:
-        # print("Function Template", c.spelling, c.raw_comment)
+#        print("Function Template", c.spelling, c.raw_comment)
         objects["functions"].append(Function(c))
         return
 
@@ -90,6 +93,18 @@ def traverse(c, path, objects):
         objects["enums"].append(Enum(c))
         return
 
+    elif c.kind == clang.cindex.CursorKind.CLASS_DECL:
+        objects["classes"].append(Class(c))
+        return
+
+    elif c.kind == clang.cindex.CursorKind.CLASS_TEMPLATE:
+        objects["classes"].append(Class(c))
+        return
+
+    elif c.kind == clang.cindex.CursorKind.STRUCT_DECL:
+        objects["structs"].append(Class(c))
+        return
+
     else:
         # print("Unknown", c.kind, c.spelling)
         pass
@@ -100,20 +115,16 @@ def traverse(c, path, objects):
 
 def parse(path):
     index = clang.cindex.Index.create()
-    tu = index.parse(path, ['-x', 'c++', '-std=c++11', '-fparse-all-comments', '-DIGL_STATIC_LIBRARY'])
-    # Clang can't parse files with missing definitions, add static library definition
-    objects = {"functions": [], "enums": [], "namespaces": [], "classes": []}
+    # Clang can't parse files with missing definitions, add static library definition or not?
+    args = ['-x', 'c++', '-std=c++11', '-fparse-all-comments', '-DIGL_STATIC_LIBRARY']
+    args.append('-I/usr/include/eigen3/') # TODO Properly add all needed includes
+    syspath = ccsyspath.system_include_paths('clang++-3.7') # Add the system libraries
+    incargs = [(b'-I' + inc).decode("utf-8") for inc in syspath]
+    args.extend(incargs)
+
+    tu = index.parse(path, args)
+    objects = {"functions": [], "enums": [], "namespaces": [], "classes": [], "structs": []}
     traverse(tu.cursor, path, objects)
-
-    #    tpl = Template(filename='bind.mako')
-    #    rendered = tpl.render(functions=functions)
-
-    #    OUTPUT_DIR = 'generated'
-
-    #    if not os.path.isdir(OUTPUT_DIR): os.mkdir(OUTPUT_DIR)
-
-    #    with open("generated/{}.bind.cc".format(sys.argv[1]), "w") as f:
-    #        f.write(rendered)
     return objects
 
 if __name__ == '__main__':