#! /usr/bin/env python # encoding: utf-8 # hhaim, 2014 (IL) base on WAF book """ call 'waf --targets=waf.pdf' or use 'waf list' to see the targets available """ VERSION='0.0.1' APPNAME='wafdocs' import os, re, shutil import sys import shlex import subprocess import json import re from waflib import Logs top = '.' out = 'build' sphinx_version = None try: from HTMLParser import HTMLParser except: from html.parser import HTMLParser class CTocNode: def __init__ (self): self.name="root" self.level=1; # 1,2,3,4 self.link=None; self.parent=None self.childs=[]; # link to CTocNode def get_link (self): if self.link==None: name=self.name l=name.split('.'); l=l[-1].lower() s=''; for c in l: if c.isalpha() or c.isspace(): s+=c return '#_'+'_'.join(s.lower().split()); else: return '#'+self.link def add_new_child (self,name,level,link): n=CTocNode(); n.name=name; n.link=link n.level=level; n.parent=self; self.childs.append(n); return n def to_json_childs (self): l=[] for obj in self.childs: l.append(obj.to_json()); return (l); def to_open (self): if self.level <3: return True else: return False def to_json (self): d={"text" : self.name, "link" : self.get_link(), "state" : { "opened" : self.to_open() } } if len(self.childs)>0 : d["children"]= self.to_json_childs() return d class TocHTMLParser(HTMLParser): def __init__ (self): HTMLParser.__init__(self); self.state=0; self.root=CTocNode() self.root.parent=self.root self.level=2; self.attrs=None self.d={}; self.last_level=1 self.set_level(1,self.root) def set_level (self,level,node): assert(node!=None); assert(isinstance(node,CTocNode)==True); self.d[str(level)]=node # in case we change from high to low level remove the higher level if level1): return (True); def handle_starttag(self, tag, attrs): if self.is_header (tag): self.attrs=attrs self.state=True; self.level=int(tag[1]); def handle_endtag(self, tag): if self.is_header (tag): self.state=False; def get_id (self): if self.attrs: for obj in self.attrs: if obj[0]=='id': return obj[1] else: return None def handle_data(self, data): if self.state: level=self.level cnode=self.get_level(level-1) n=cnode.add_new_child(data,level,self.get_id()); assert(n!=None); self.set_level(level,n) self.last_level=level def dump_as_json (self): return json.dumps(self.root.to_json_childs(), sort_keys=False, indent=4) def create_toc_json (input_file,output_file): f = open (input_file) l=f.readlines() f.close(); html_input = ''.join(l) parser = TocHTMLParser() parser.feed(html_input); f = open (output_file,'w') f.write(parser.dump_as_json()); f.close(); re_xi = re.compile('''^(include|image)::([^.]*.(asciidoc|\\{PIC\\}))\[''', re.M) def ascii_doc_scan(self): p = self.inputs[0].parent node_lst = [self.inputs[0]] seen = [] depnodes = [] while node_lst: nd = node_lst.pop(0) if nd in seen: continue seen.append(nd) code = nd.read() for m in re_xi.finditer(code): name = m.group(2) if m.group(3) == '{PIC}': ext = '.eps' if self.generator.rule.rfind('A2X') > 0: ext = '.png' k = p.find_resource(name.replace('{PIC}', ext)) if k: depnodes.append(k) else: k = p.find_resource(name) if k: depnodes.append(k) node_lst.append(k) return [depnodes, ()] def scansize(self): name = 'image::%s\\{PIC\\}\\[.*,(width|height)=(\\d+)' % self.inputs[0].name[:-4] re_src = re.compile(name) lst = self.inputs[0].parent.get_src().ant_glob('*.txt') for x in lst: m = re_src.search(x.read()) if m: val = str(int(1.6 * int(m.group(2)))) if m.group(1) == 'width': w = val h = "800" else: w = "800" h = val ext = self.inputs[0].name[-3:] if ext == 'eps': code = '-geometry %sx%s' % (w, h) elif ext == 'dia': if m.group(1) == 'width': h = '' else: w = '' code = '--size %sx%s' % (w, h) else: code = '-Gsize="%s,%s"' % (w, h) break else: return ([], '') return ([], code) def options(opt): opt.add_option('--exe', action='store_true', default=False, help='Execute the program after it is compiled') def configure(conf): search_path = '~/.local/bin /usr/local/bin/ /usr/bin' conf.find_program('asciidoc', path_list=search_path, var='ASCIIDOC') conf.find_program('sphinx-build', path_list=search_path, var='SPHINX') conf.find_program('source-highlight', path_list=search_path, var='SRC_HIGHLIGHT') conf.find_program('dblatex', path_list=search_path, var='DBLATEX') conf.find_program('a2x', path_list=search_path, var='A2X') pass; def convert_to_pdf(task): input_file = task.outputs[0].abspath() out_dir = task.outputs[0].parent.get_bld().abspath() return os.system('a2x --no-xmllint %s -f pdf -d article %s -D %s ' %('-v' if Log.verbose else '', task.inputs[0].abspath(),out_dir ) ) TOC_HEAD = """
Table of Contents
""" TOC_END = """
""" def do_replace (input_file,contents,look,str_replaced): if contents.count(look)!=1 : raise Exception('Cannot find {0} in file {1} '.format(look,input_file)) return contents.replace(look, str_replaced) def toc_fixup_file (input_file, out_file, json_file_name ): file = open(input_file) contents = file.read() contents = do_replace(input_file,contents,'', TOC_HEAD); contents = do_replace(input_file,contents,'', TOC_END) contents = do_replace(input_file,contents,'input_replace_me.json', json_file_name) file = open(out_file,'w') file.write(contents) file.close(); def convert_to_html_toc_book(task): input_file = task.inputs[0].abspath() json_out_file = os.path.splitext(task.outputs[0].abspath())[0]+'.json' tmp = os.path.splitext(task.outputs[0].abspath())[0]+'.tmp' json_out_file_short = os.path.splitext(task.outputs[0].name)[0]+'.json' cmd='{0} -a stylesheet={1} -a icons=true -a docinfo -d book -o {2} {3}'.format( task.env['ASCIIDOC'][0], task.inputs[1].abspath(), tmp, task.inputs[0].abspath()); res= os.system( cmd ) if res !=0 : return (1) create_toc_json(tmp,json_out_file) toc_fixup_file(tmp,task.outputs[0].abspath(),json_out_file_short); return os.system('rm {0}'.format(tmp)); def convert_to_pdf_book(task): input_file = task.outputs[0].abspath() out_dir = task.outputs[0].parent.get_bld().abspath() return os.system('a2x --no-xmllint %s -f pdf -d book %s -D %s ' %('-v' if Logs.verbose else '', task.inputs[0].abspath(),out_dir ) ) def ensure_dir(f): if not os.path.exists(f): os.makedirs(f) def my_copy(task): input_file=task.outputs[0].abspath() out_dir=task.outputs[0].parent.get_bld().abspath() ensure_dir(out_dir) shutil.copy2(input_file, out_dir+ os.sep+task.outputs[0].name) return (0) def do_visio(bld): for x in bld.path.ant_glob('visio\\*.vsd'): tg = bld(rule='${VIS} -i ${SRC} -o ${TGT} ', source=x, target=x.change_ext('.png')) def get_sphinx_version(sphinx_path): try: global sphinx_version if not sphinx_version: sphinx_version_regexp = '^Sphinx \(sphinx-build\) (\d+)\.(\d+)\.\d+$' cmd = '%s %s --version' % (sys.executable, sphinx_path) output = subprocess.check_output(shlex.split(cmd), universal_newlines = True) for line in output.splitlines(): ver = re.match(sphinx_version_regexp, line) if ver: sphinx_version = float('%s.%s' % (ver.group(1), ver.group(2))) return sphinx_version except Exception as e: print('Error getting Sphinx version: %s' % e) def get_trex_core_git(): trex_core_git_path = os.path.abspath(os.path.join(os.getcwd(), os.pardir)) if not os.path.isdir(trex_core_git_path): trex_core_git_path = os.getenv('TREX_CORE_GIT', None) return trex_core_git_path def parse_hlt_args(task): trex_core_git_path = get_trex_core_git() if not trex_core_git_path: return 1 hltapi_path = os.path.abspath(os.path.join(trex_core_git_path, 'scripts', 'automation', 'trex_control_plane', 'stl', 'trex_stl_lib', 'trex_stl_hltapi.py')) header = ['[options="header",cols="<.^1,^.^1,9<.^e"]', '|=================', '^| Argument | Default ^| Comment'] footer = ['|=================\n'] hlt_asciidoc = [] category_regexp = '^(\S+)_kwargs = {$' comment_line_regexp = '^\s*#\s*(.+)$' arg_line_regexp = "^\s*'([^']+)':\s*'?([^,']+)'?,\s*#?\s*(.+)?$" if not os.path.exists(hltapi_path): raise Exception('Could not find hltapi file: %s' % hltapi_path) with open(hltapi_path) as f: in_args = False for line in f.read().splitlines(): if not in_args: if line.startswith('import'): break category_line = re.match(category_regexp, line) if category_line: hlt_asciidoc.append('\n===== %s\n' % category_line.group(1)) hlt_asciidoc += header in_args = True continue comment_line = re.match(comment_line_regexp, line) if comment_line: hlt_asciidoc.append('3+^.^s| %s' % comment_line.group(1).replace('|', '\|')) continue arg_line = re.match(arg_line_regexp, line) if arg_line: arg, default, comment = arg_line.groups() hlt_asciidoc.append('| %s | %s | %s' % (arg, default, comment.replace('|', '\|') if comment else '')) continue if line == '}': hlt_asciidoc += footer in_args = False if not len(hlt_asciidoc): raise Exception('Parsing of hltapi args failed') with open('build/hlt_args.asciidoc', 'w') as f: f.write('\n'.join(hlt_asciidoc)) return 0 def build_cp_docs (task): out_dir = task.outputs[0].abspath() export_path = os.path.join(os.getcwd(), 'build', 'cp_docs') trex_core_git_path = get_trex_core_git() if not trex_core_git_path: # there exists a default directory or the desired ENV variable. return 1 trex_core_docs_path = os.path.abspath(os.path.join(trex_core_git_path, 'scripts', 'automation', 'trex_control_plane', 'doc')) sphinx_version = get_sphinx_version(task.env['SPHINX'][0]) if not sphinx_version: return 1 if sphinx_version < 1.3: additional_args = '-D html_theme=default' else: additional_args = '' build_doc_cmd = "{pyt} {sph} {add} {ver} -W -b {bld} {src} {dst}".format( pyt= sys.executable, sph= task.env['SPHINX'][0], add= additional_args, ver= '' if Logs.verbose else '-q', bld= "html", src= ".", dst= out_dir) if Logs.verbose: print(build_doc_cmd) try: return subprocess.call(shlex.split(build_doc_cmd), cwd = trex_core_docs_path) except OSError as e: print('Failed command: %s\nError: %s' % (build_doc_cmd, e)) return 1 def build_stl_cp_docs (task): out_dir = task.outputs[0].abspath() export_path = os.path.join(os.getcwd(), 'build', 'cp_stl_docs') trex_core_git_path = get_trex_core_git() if not trex_core_git_path: # there exists a default directory or the desired ENV variable. return 1 trex_core_docs_path = os.path.abspath(os.path.join(trex_core_git_path, 'scripts', 'automation', 'trex_control_plane', 'doc_stl')) sphinx_version = get_sphinx_version(task.env['SPHINX'][0]) if not sphinx_version: return 1 if sphinx_version < 1.3: additional_args = '-D html_theme=default' else: additional_args = '' build_doc_cmd = "{pyt} {sph} {add} {ver} -W -b {bld} {src} {dst}".format( pyt= sys.executable, sph= task.env['SPHINX'][0], add= additional_args, ver= '' if Logs.verbose else '-q', bld= "html", src= ".", dst= out_dir) if Logs.verbose: print(build_doc_cmd) try: return subprocess.call(shlex.split(build_doc_cmd), cwd = trex_core_docs_path) except OSError as e: print('Failed command: %s\nError: %s' % (build_doc_cmd, e)) return 1 def build_cp(bld,dir,root,callback): export_path = os.path.join(os.getcwd(), 'build', dir) trex_core_git_path = get_trex_core_git() if not trex_core_git_path: # there exists a default directory or the desired ENV variable. raise NameError("Environment variable 'TREX_CORE_GIT' is not defined.") trex_core_docs_path = os.path.join(trex_core_git_path, 'scripts', 'automation', 'trex_control_plane', root, 'index.rst') bld(rule=callback,target = dir) def build(bld): bld(rule=my_copy, target='symbols.lang') for x in bld.path.ant_glob('images\\**\**.png'): bld(rule=my_copy, target=x) bld.add_group() for x in bld.path.ant_glob('yaml\\**\**.yaml'): bld(rule=my_copy, target=x) bld.add_group() for x in bld.path.ant_glob('video\\**\**.mp4'): bld(rule=my_copy, target=x) bld.add_group() for x in bld.path.ant_glob('images\\**\**.jpg'): bld(rule=my_copy, target=x) bld.add_group() bld(rule=my_copy, target='my_chart.js') build_cp(bld,'hlt_args.asciidoc','stl/trex_stl_lib', parse_hlt_args) bld.add_group() # separator, the documents may require any of the pictures from above if os.path.exists('build/hlt_args.asciidoc'): bld.add_manual_dependency( bld.path.find_node('trex_stateless.asciidoc'), b'build/hlt_args.asciidoc') bld(rule='${ASCIIDOC} -f ../backends/deckjs/deckjs.conf -o ${TGT} ${SRC[0].abspath()}', source='trex_config.asciidoc ', target='trex_config_guide.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -f ../backends/deckjs/deckjs.conf -o ${TGT} ${SRC[0].abspath()}', source='trex_preso.asciidoc ', target='trex_preso.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='release_notes.asciidoc waf.css', target='release_notes.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a docinfo -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -d book -o ${TGT} ${SRC[0].abspath()}', source='draft_trex_stateless.asciidoc waf.css', target='draft_trex_stateless.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a docinfo -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -d book -o ${TGT} ${SRC[0].abspath()}', source='draft_trex_stateless_moved1.asciidoc waf.css', target='draft_trex_stateless1.html', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book,source='trex_book.asciidoc waf.css', target='trex_book.pdf', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book,source='trex_stateless.asciidoc waf.css', target='trex_stateless.pdf', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book,source='draft_trex_stateless.asciidoc waf.css', target='draft_trex_stateless.pdf', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book,source='trex_vm_manual.asciidoc waf.css', target='trex_vm_manual.pdf', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book,source='trex_control_plane_peek.asciidoc waf.css', target='trex_control_plane_peek.pdf', scan=ascii_doc_scan) bld(rule=convert_to_pdf_book, source='trex_control_plane_design_phase1.asciidoc waf.css', target='trex_control_plane_design_phase1.pdf', scan=ascii_doc_scan) # with nice TOC bld(rule=convert_to_html_toc_book, source='trex_vm_manual.asciidoc waf.css', target='trex_vm_manual.html',scan=ascii_doc_scan) bld(rule=convert_to_html_toc_book, source='trex_stateless.asciidoc waf.css', target='trex_stateless.html',scan=ascii_doc_scan); bld(rule=convert_to_html_toc_book, source='trex_stateless_bench.asciidoc waf.css', target='trex_stateless_bench.html',scan=ascii_doc_scan); bld(rule=convert_to_html_toc_book, source='trex_book.asciidoc waf.css', target='trex_manual.html',scan=ascii_doc_scan); bld(rule=convert_to_html_toc_book, source='trex_faq.asciidoc waf.css', target='trex_faq.html',scan=ascii_doc_scan); bld(rule=convert_to_html_toc_book, source='trex_rpc_server_spec.asciidoc waf.css', target='trex_rpc_server_spec.html',scan=ascii_doc_scan); bld(rule=convert_to_html_toc_book, source='trex_scapy_rpc_server.asciidoc waf.css', target='trex_scapy_rpc_server.html',scan=ascii_doc_scan); bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='vm_doc.asciidoc waf.css', target='vm_doc.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='packet_builder_yaml.asciidoc waf.css', target='packet_builder_yaml.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='trex_control_plane_design_phase1.asciidoc waf.css', target='trex_control_plane_design_phase1.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='trex_control_plane_peek.asciidoc waf.css', target='trex_control_plane_peek.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='trex_console.asciidoc waf.css', target='trex_console.html', scan=ascii_doc_scan) bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', source='trex_index.asciidoc waf.css', target='index.html', scan=ascii_doc_scan) build_cp(bld,'cp_docs','doc',build_cp_docs) build_cp(bld,'cp_stl_docs','doc_stl',build_stl_cp_docs) class Env(object): @staticmethod def get_env(name) : s= os.environ.get(name); if s == None: print("You should define $",name) raise Exception("Env error"); return (s); @staticmethod def get_release_path () : s= Env().get_env('TREX_LOCAL_PUBLISH_PATH'); s +=get_build_num ()+"/" return s; @staticmethod def get_remote_release_path () : s= Env().get_env('TREX_REMOTE_PUBLISH_PATH'); return s; @staticmethod def get_local_web_server () : s= Env().get_env('TREX_WEB_SERVER'); return s; # extral web @staticmethod def get_trex_ex_web_key() : s= Env().get_env('TREX_EX_WEB_KEY'); return s; @staticmethod def get_trex_ex_web_path() : s= Env().get_env('TREX_EX_WEB_PATH'); return s; @staticmethod def get_trex_ex_web_user() : s= Env().get_env('TREX_EX_WEB_USER'); return s; @staticmethod def get_trex_ex_web_srv() : s= Env().get_env('TREX_EX_WEB_SRV'); return s; @staticmethod def get_trex_core() : s= Env().get_env('TREX_CORE_GIT'); return s; def release(bld): # copy all the files to our web server core_dir = Env().get_trex_core() release_dir = core_dir +"/scripts/doc/"; os.system('mkdir -p '+release_dir) os.system('cp -rv build/release_notes.* '+ release_dir) def publish(bld): # copy all the files to our web server remote_dir = "%s:%s" % ( Env().get_local_web_server(), Env().get_remote_release_path ()+'../doc/') os.system('rsync -av --del --rsh=ssh build/ %s' % (remote_dir)) def publish_ext(bld): from_ = 'build/' os.system('rsync -avz --del -e "ssh -i %s" --rsync-path=/usr/bin/rsync %s %s@%s:%s/doc/' % (Env().get_trex_ex_web_key(),from_, Env().get_trex_ex_web_user(),Env().get_trex_ex_web_srv(),Env().get_trex_ex_web_path() ) ) def publish_test(bld): # copy all the files to our web server remote_dir = "%s:%s" % ( Env().get_local_web_server(), Env().get_remote_release_path ()+'../test/') os.system('rsync -av --del --rsh=ssh build/ %s' % (remote_dir)) def publish_both(bld): publish(bld) publish_ext(bld) def test(bld): # copy all the files to our web server toc_fixup_file ('build/trex_stateless.tmp', 'build/trex_stateless.html', 'trex_stateless.json')