xen-project/xen
1
0 Fork 1
mirror of https://github.com/xen-project/xen.git synced 2026-10-28 10:11:27 +01:00
Code Issues Activity
Files
b7d4a69f0ccb5b6035d5b567694bf2c2dc5983c4
xen/docs/pythfilter.py
T
ewan@linford.intra b7d4a69f0c "Added."
2005-09-11 21:40:15 +01:00

640 lines
21 KiB
Python
Raw Blame History

#!/usr/bin/env python
# pythfilter.py v1.5.5, written by Matthias Baas (baas@ira.uka.de)
# Doxygen filter which can be used to document Python source code.
# Classes (incl. methods) and functions can be documented.
# Every comment that begins with ## is literally turned into an
# Doxygen comment. Consecutive comment lines are turned into
# comment blocks (-> /** ... */).
# All the stuff is put inside a namespace with the same name as
# the source file.
# Conversions:
# ============
# ##-blocks -> /** ... */
# "class name(base): ..." -> "class name : public base {...}"
# "def name(params): ..." -> "name(params) {...}"
# Changelog:
# 21.01.2003: Raw (r"") or unicode (u"") doc string will now be properly
# handled. (thanks to Richard Laager for the patch)
# 22.12.2003: Fixed a bug where no function names would be output for "def"
# blocks that were not in a class.
# (thanks to Richard Laager for the patch)
# 12.12.2003: Implemented code to handle static and class methods with
# this logic: Methods with "self" as the first argument are
# non-static. Methods with "cls" are Python class methods,
# which translate into static methods for Doxygen. Other
# methods are assumed to be static methods. As should be
# obvious, this logic doesn't take into account if the method
# is actually setup as a classmethod() or a staticmethod(),
# just if it follows the normal conventions.
# (thanks to Richard Laager for the patch)
# 11.12.2003: Corrected #includes to use os.path.sep instead of ".". Corrected
# namespace code to use "::" instead of ".".
# (thanks to Richard Laager for the patch)
# 11.12.2003: Methods beginning with two underscores that end with
# something other than two underscores are considered private
# and are handled accordingly.
# (thanks to Richard Laager for the patch)
# 03.12.2003: The first parameter of class methods (self) is removed from
# the documentation.
# 03.11.2003: The module docstring will be used as namespace documentation
# (thanks to Joe Bronkema for the patch)
# 08.07.2003: Namespaces get a default documentation so that the namespace
# and its contents will show up in the generated documentation.
# 05.02.2003: Directories will be delted during synchronization.
# 31.01.2003: -f option & filtering entire directory trees.
# 10.08.2002: In base classes the '.' will be replaced by '::'
# 18.07.2002: * and ** will be translated into arguments
# 18.07.2002: Argument lists may contain default values using constructors.
# 18.06.2002: Support for ## public:
# 21.01.2002: from ... import will be translated to "using namespace ...;"
# TODO: "from ... import *" vs "from ... import names"
# TODO: Using normal imports: name.name -> name::name
# 20.01.2002: #includes will be placed in front of the namespace
######################################################################
# The program is written as a state machine with the following states:
#
# - OUTSIDE The current position is outside any comment,
# class definition or function.
#
# - BUILD_COMMENT Begins with first "##".
# Ends with the first token that is no "##"
# at the same column as before.
#
# - BUILD_CLASS_DECL Begins with "class".
# Ends with ":"
# - BUILD_CLASS_BODY Begins just after BUILD_CLASS_DECL.
# The first following token (which is no comment)
# determines indentation depth.
# Ends with a token that has a smaller indendation.
#
# - BUILD_DEF_DECL Begins with "def".
# Ends with ":".
# - BUILD_DEF_BODY Begins just after BUILD_DEF_DECL.
# The first following token (which is no comment)
# determines indentation depth.
# Ends with a token that has a smaller indendation.
import getopt
import glob
import os.path
import re
import shutil
import string
import sys
import token
import tokenize
from stat import *
OUTSIDE = 0
BUILD_COMMENT = 1
BUILD_CLASS_DECL = 2
BUILD_CLASS_BODY = 3
BUILD_DEF_DECL = 4
BUILD_DEF_BODY = 5
IMPORT = 6
IMPORT_OP = 7
IMPORT_APPEND = 8
# Output file stream
outfile = sys.stdout
# Output buffer
outbuffer = []
out_row = 1
out_col = 0
# Variables used by rec_name_n_param()
name = ""
param = ""
doc_string = ""
record_state = 0
bracket_counter = 0
# Tuple: (row,column)
class_spos = (0,0)
def_spos = (0,0)
import_spos = (0,0)
# Which import was used? ("import" or "from")
import_token = ""
# Comment block buffer
comment_block = []
comment_finished = 0
# Imported modules
modules = []
# Program state
stateStack = [OUTSIDE]
# Keep track of whether module has a docstring
module_has_docstring = False
# Keep track of member protection
protection_level = "public"
private_member = False
# Keep track of the module namespace
namespace = ""
######################################################################
# Output string s. '\n' may only be at the end of the string (not
# somewhere in the middle).
#
# In: s - String
# spos - Startpos
######################################################################
def output(s,spos, immediate=0):
global outbuffer, out_row, out_col, outfile
os = string.rjust(s,spos[1]-out_col+len(s))
if immediate:
outfile.write(os)
else:
outbuffer.append(os)
assert -1 == string.find(s[0:-2], "\n"), s
if (s[-1:]=="\n"):
out_row = out_row+1
out_col = 0
else:
out_col = spos[1]+len(s)
######################################################################
# Records a name and parameters. The name is either a class name or
# a function name. Then the parameter is either the base class or
# the function parameters.
# The name is stored in the global variable "name", the parameters
# in "param".
# The variable "record_state" holds the current state of this internal
# state machine.
# The recording is started by calling start_recording().
#
# In: type, tok
######################################################################
def rec_name_n_param(type, tok):
global record_state,name,param,doc_string,bracket_counter
s = record_state
# State 0: Do nothing.
if (s==0):
return
# State 1: Remember name.
elif (s==1):
name = tok
record_state = 2
# State 2: Wait for opening bracket or colon
elif (s==2):
if (tok=='('):
bracket_counter = 1
record_state=3
if (tok==':'): record_state=4
# State 3: Store parameter (or base class) and wait for an ending bracket
elif (s==3):
if (tok=='*' or tok=='**'):
tok=''
if (tok=='('):
bracket_counter = bracket_counter+1
if (tok==')'):
bracket_counter = bracket_counter-1
if bracket_counter==0:
record_state=4
else:
param=param+tok
# State 4: Look for doc string
elif (s==4):
if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
return
elif (tok==":"):
return
elif (type==token.STRING):
while tok[:1]=='r' or tok[:1]=='u':
tok=tok[1:]
while tok[:1]=='"':
tok=tok[1:]
while tok[-1:]=='"':
tok=tok[:-1]
doc_string=tok
record_state=0
######################################################################
# Starts the recording of a name & param part.
# The function rec_name_n_param() has to be fed with tokens. After
# the necessary tokens are fed the name and parameters can be found
# in the global variables "name" und "param".
######################################################################
def start_recording():
global record_state,param,name, doc_string
record_state=1
name=""
param=""
doc_string=""
######################################################################
# Test if recording is finished
######################################################################
def is_recording_finished():
global record_state
return record_state==0
######################################################################
## Gather comment block
######################################################################
def gather_comment(type,tok,spos):
global comment_block,comment_finished
if (type!=tokenize.COMMENT):
comment_finished = 1
else:
# Output old comment block if a new one is started.
if (comment_finished):
print_comment(spos)
comment_finished=0
if (tok[0:2]=="##" and tok[0:3]!="###"):
append_comment_lines(tok[2:])
######################################################################
## Output comment block and empty buffer.
######################################################################
def print_comment(spos):
global comment_block,comment_finished
if (comment_block!=[]):
output("/** ",spos)
for c in comment_block:
output(c,spos)
output("*/\n",spos)
comment_block = []
comment_finished = 0
######################################################################
def set_state(s):
global stateStack
stateStack[len(stateStack)-1]=s
######################################################################
def get_state():
global stateStack
return stateStack[len(stateStack)-1]
######################################################################
def push_state(s):
global stateStack
stateStack.append(s)
######################################################################
def pop_state():
global stateStack
stateStack.pop()
######################################################################
def tok_eater(type, tok, spos, epos, line):
global stateStack,name,param,class_spos,def_spos,import_spos
global doc_string, modules, import_token, module_has_docstring
global protection_level, private_member
global out_row
while out_row + 1 < spos[0]:
output("\n", (0, 0))
rec_name_n_param(type,tok)
if (string.replace(string.strip(tok)," ","")=="##private:"):
protection_level = "private"
output("private:\n",spos)
elif (string.replace(string.strip(tok)," ","")=="##protected:"):
protection_level = "protected"
output("protected:\n",spos)
elif (string.replace(string.strip(tok)," ","")=="##public:"):
protection_level = "public"
output("public:\n",spos)
else:
gather_comment(type,tok,spos)
state = get_state()
# sys.stderr.write("%d: %s\n"%(state, tok))
# OUTSIDE
if (state==OUTSIDE):
if (tok=="class"):
start_recording()
class_spos = spos
push_state(BUILD_CLASS_DECL)
elif (tok=="def"):
start_recording()
def_spos = spos
push_state(BUILD_DEF_DECL)
elif (tok=="import") or (tok=="from"):
import_token = tok
import_spos = spos
modules = []
push_state(IMPORT)
elif (spos[1] == 0 and tok[:3] == '"""'):
# Capture module docstring as namespace documentation
module_has_docstring = True
append_comment_lines("\\namespace %s\n" % namespace)
append_comment_lines(tok[3:-3])
print_comment(spos)
# IMPORT
elif (state==IMPORT):
if (type==token.NAME):
modules.append(tok)
set_state(IMPORT_OP)
# IMPORT_OP
elif (state==IMPORT_OP):
if (tok=="."):
set_state(IMPORT_APPEND)
elif (tok==","):
set_state(IMPORT)
else:
for m in modules:
output('#include "'+m.replace('.',os.path.sep)+'.py"\n', import_spos, immediate=1)
if import_token=="from":
output('using namespace '+m.replace('.', '::')+';\n', import_spos)
pop_state()
# IMPORT_APPEND
elif (state==IMPORT_APPEND):
if (type==token.NAME):
modules[len(modules)-1]+="."+tok
set_state(IMPORT_OP)
# BUILD_CLASS_DECL
elif (state==BUILD_CLASS_DECL):
if (is_recording_finished()):
s = "class "+name
if (param!=""): s = s+" : public "+param.replace('.','::')
if (doc_string!=""):
append_comment_lines(doc_string)
print_comment(class_spos)
output(s+"\n",class_spos)
output("{\n",(class_spos[0]+1,class_spos[1]))
protection_level = "public"
output(" public:\n",(class_spos[0]+2,class_spos[1]))
set_state(BUILD_CLASS_BODY)
# BUILD_CLASS_BODY
elif (state==BUILD_CLASS_BODY):
if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
type!=tokenize.NL and type!=tokenize.COMMENT and
(spos[1]<=class_spos[1])):
output("}; // end of class\n",(out_row+1,class_spos[1]))
pop_state()
elif (tok=="def"):
start_recording()
def_spos = spos
push_state(BUILD_DEF_DECL)
# BUILD_DEF_DECL
elif (state==BUILD_DEF_DECL):
if (is_recording_finished()):
param = param.replace("\n", " ")
param = param.replace("=", " = ")
params = param.split(",")
if BUILD_CLASS_BODY in stateStack:
if len(name) > 1 \
and name[0:2] == '__' \
and name[len(name)-2:len(name)] != '__' \
and protection_level != 'private':
private_member = True
output(" private:\n",(def_spos[0]+2,def_spos[1]))
if (doc_string != ""):
append_comment_lines(doc_string)
print_comment(def_spos)
output_function_decl(name, params)
# output("{\n",(def_spos[0]+1,def_spos[1]))
set_state(BUILD_DEF_BODY)
# BUILD_DEF_BODY
elif (state==BUILD_DEF_BODY):
if (type!=token.INDENT and type!=token.NEWLINE \
and type!=40 and type!=tokenize.NL \
and (spos[1]<=def_spos[1])):
# output("} // end of method/function\n",(out_row+1,def_spos[1]))
if private_member and protection_level != 'private':
private_member = False
output(" " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
pop_state()
# else:
# output(tok,spos)
def output_function_decl(name, params):
global def_spos
# Do we document a class method? then remove the 'self' parameter
if params[0] == 'self':
preamble = ''
params = params[1:]
else:
preamble = 'static '
if params[0] == 'cls':
params = params[1:]
param_string = string.join(params, ", Type ")
if param_string == '':
param_string = '(' + param_string + ');\n'
else:
param_string = '(Type ' + param_string + ');\n'
output(preamble, def_spos)
output(name, def_spos)
output(param_string, def_spos)
def append_comment_lines(lines):
map(append_comment_line, doc_string.split('\n'))
paramRE = re.compile(r'(@param \w+):')
def append_comment_line(line):
global paramRE
comment_block.append(paramRE.sub(r'\1', line) + '\n')
def dump(filename):
f = open(filename)
r = f.readlines()
for s in r:
sys.stdout.write(s)
def filter(filename):
global name, module_has_docstring
path,name = os.path.split(filename)
root,ext = os.path.splitext(name)
output("namespace "+root+" {\n",(0,0))
# set module name for tok_eater to use if there's a module doc string
name = root
# sys.stderr.write('Filtering "'+filename+'"...')
f = open(filename)
tokenize.tokenize(f.readline, tok_eater)
f.close()
print_comment((0,0))
output("\n",(0,0))
output("} // end of namespace\n",(0,0))
if not module_has_docstring:
# Put in default namespace documentation
output('/** \\namespace '+root+' \n',(0,0))
output(' \\brief Module "%s" */\n'%(root),(0,0))
for s in outbuffer:
outfile.write(s)
def filterFile(filename, out=sys.stdout):
global outfile
outfile = out
try:
root,ext = os.path.splitext(filename)
if ext==".py":
filter(filename)
else:
dump(filename)
# sys.stderr.write("OK\n")
except IOError,e:
sys.stderr.write(e[1]+"\n")
######################################################################
# preparePath
def preparePath(path):
"""Prepare a path.
Checks if the path exists and creates it if it does not exist.
"""
if not os.path.exists(path):
parent = os.path.dirname(path)
if parent!="":
preparePath(parent)
os.mkdir(path)
# isNewer
def isNewer(file1,file2):
"""Check if file1 is newer than file2.
file1 must be an existing file.
"""
if not os.path.exists(file2):
return True
return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
# convert
def convert(srcpath, destpath):
"""Convert a Python source tree into a C+ stub tree.
All *.py files in srcpath (including sub-directories) are filtered
and written to destpath. If destpath exists, only the files
that have been modified are filtered again. Files that were deleted
from srcpath are also deleted in destpath if they are still present.
The function returns the number of processed *.py files.
"""
count=0
sp = os.path.join(srcpath,"*")
sfiles = glob.glob(sp)
dp = os.path.join(destpath,"*")
dfiles = glob.glob(dp)
leftovers={}
for df in dfiles:
leftovers[os.path.basename(df)]=1
for srcfile in sfiles:
basename = os.path.basename(srcfile)
if basename in leftovers:
del leftovers[basename]
# Is it a subdirectory?
if os.path.isdir(srcfile):
sdir = os.path.join(srcpath,basename)
ddir = os.path.join(destpath,basename)
count+=convert(sdir, ddir)
continue
# Check the extension (only *.py will be converted)
root, ext = os.path.splitext(srcfile)
if ext.lower()!=".py":
continue
destfile = os.path.join(destpath,basename)
if destfile==srcfile:
print "WARNING: Input and output names are identical!"
sys.exit(1)
count+=1
# sys.stdout.write("%s\015"%(srcfile))
if isNewer(srcfile, destfile):
preparePath(os.path.dirname(destfile))
# out=open(destfile,"w")
# filterFile(srcfile, out)
# out.close()
os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
# Delete obsolete files in destpath
for df in leftovers:
dname=os.path.join(destpath,df)
if os.path.isdir(dname):
try:
shutil.rmtree(dname)
except:
print "Can't remove obsolete directory '%s'"%dname
else:
try:
os.remove(dname)
except:
print "Can't remove obsolete file '%s'"%dname
return count
######################################################################
######################################################################
######################################################################
filter_file = False
try:
opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
except getopt.GetoptError,e:
print e
sys.exit(1)
for o,a in opts:
if o=="-f":
filter_file = True
if filter_file:
# Filter the specified file and print the result to stdout
filename = string.join(args)
filterFile(filename)
else:
if len(args)!=2:
sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
sys.exit(1)
# Filter an entire Python source tree
print '"%s" -> "%s"\n'%(args[0],args[1])
c=convert(args[0],args[1])
print "%d files"%(c)
Reference in new issue View Git Blame Copy Permalink