#@+leo-ver=5-thin
#@+node:ekr.20170302123956.1: * @file ../doc/leoAttic.txt
# This is Leo's final resting place for dead code.
# Much easier to access than a git repo.

#@@language python
#@@killbeautify
#@+all
#@+node:ekr.20191029023442.1: **  4 kinds of documentation
@language rest
@wrap

A tutorial: (learn by doing: teacher guides)
    is learning-oriented
    allows the newcomer to get started
    is a lesson
    
A how-to guide: (recipe: users ask questions)
    is goal-oriented
    shows how to solve a specific problem
    is a series of steps

A reference guide:
    is information-oriented
    describes the machinery
    is accurate and complete

An explanation/discussion
    is understanding-oriented
    explains
    provides background and context
#@+node:ekr.20190412154439.1: ** Abandoned projects
#@+node:ekr.20170203080350.1: *3* Abandoned #396: Show images in Leo's body pane
https://github.com/leo-editor/leo-editor/issues/396

#@+node:ekr.20170302151109.1: *4* ** Notes
@language rest
@wrap

Unicode 'object replacement character': u+FFFC

QTextDocument may be helpful: http://doc.qt.io/qt-5/qtextdocument.html
See QTextDocument.MetaInformation: http://doc.qt.io/qt-5/qtextdocument.html#MetaInformation-enum

http://stackoverflow.com/questions/3254652/
several-ways-of-placing-an-image-in-a-qtextedit

http://doc.qt.io/qt-5/qtextdocument.html#resource

QVariant QTextDocument::resource(int type, const QUrl &name) const

Returns data of the specified type from the resource with the given name.

This function is called by the rich text engine to request data that isn't directly stored by QTextDocument, but still associated with it. For example, images are referenced indirectly by the name attribute of a QTextImageFormat object.

Resources are cached internally in the document. If a resource can not be found in the cache, loadResource is called to try to load the resource. loadResource should then use addResource to add the resource to the cache.
#@+node:ekr.20170203105538.1: *4* Test inserting picture (new)
# https://github.com/leo-editor/leo-editor/issues/396
g.cls()
images = [i for i in range(len(p.b)) if ord(p.b[i]) > 128]
if images:
    print('images at', images)
else:
    table = (
        'application-x-leo-outline.png',
        'LeoDoc.ico',
    )    
    for image in table:
        path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
        assert g.os_path_exists(path), repr(path)
        c.frame.body.wrapper.setInsertPoint(len(p.b))
        if 0:
            format = QtGui.QTextImageFormat()
            format.setName(path)
            cursor = cursor = body.widget.textCursor()
            cursor.insertImage(format)
        c.frame.body.widget.insertHtml('<img src="%s">' % path)
            # style="width:40px;height:80px;"
    for i, ch in enumerate(p.b):
        if ord(ch) > 128: print('new', i, ord(ch))
#
#￼￼
#@+node:ekr.20170204110006.1: *4* test3 insert an image
g.cls()
from leo.core.leoQt import QtGui
body = c.frame.body
table = ('box01.bmp','box02.bmp','box03.bmp',)
d = g.app.permanentScriptDict
images = d.get('images', [])
for image in table:
    path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
    assert g.os_path_exists(path), repr(path)
    image = QtGui.QImage(path)
    images.append(image)
    body.wrapper.setInsertPoint(len(p.b))
    cursor = body.widget.textCursor()
    cursor.insertImage(image)
for i, ch in enumerate(p.b):
    if ord(ch) > 128: print('new', i, ord(ch))
d ['images'] = images
#
#
#@+node:ekr.20170204140521.1: *4* clear g.app.permanentScriptDict
g.printDict(g.app.permanentScriptDict)
g.app.permanentScriptDict = {}
#@+node:ekr.20170204135338.1: *4* @@button show-images
from leo.core.leoQt import QtCore
d = g.app.permanentScriptDict
name_index = d.get('name_index', 0)
names = ['leo_image%s' % (i) for i in range(name_index)]
# print('image names', names)
widget = c.frame.body.widget
doc = widget.document()
for i, name in enumerate(names):
    image = doc.resource(doc.ImageResource, QtCore.QUrl(name))
    print(name, image)
#@+node:ekr.20170204105958.1: *4* test2 insert an image
g.cls()
from leo.core.leoQt import QtGui
body = c.frame.body
table = (
    'application-x-leo-outline.png',
    'LeoDoc.ico',
)
g.app.permanentScriptDict = {}
d = g.app.permanentScriptDict
images = d.get('images', [])
cursors = d.get('cursors', [])
name_index = d.get('name_index', 0)
for image in table:
    path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
    assert g.os_path_exists(path), repr(path)
    image = QtGui.QImage(path)
    print(image.format().name())
    images.append(image)
    body.wrapper.setInsertPoint(len(p.b))
    cursor = body.widget.textCursor()
    #name = 'leo_image%s' % name_index
    #name_index += 1
    cursor.insertImage(image)
    cursors.append(cursor)
for i, ch in enumerate(p.b):
    if ord(ch) > 128: print('new', i, ord(ch))
d ['images'] = images
d ['cursors'] = cursors
d ['name_index'] = name_index
g.app.permanentScriptDict = d
g.printDict(d)
#
#￼￼￼
#￼
#@+node:ekr.20190910020834.1: *3* black
#@+node:ekr.20190910023018.1: *4* imports
try:
    # pylint: disable=import-error
        # We can't assume the user has this.
    import black
except Exception:
    black = None
#@+node:ekr.20150531042746.1: *4* munging leo directives
#@+node:ekr.20150529084212.1: *5* comment_leo_lines (leoBeautifier.py)
def comment_leo_lines(p):
    '''Replace lines with Leonine syntax with special comments.'''
    # Choose the comment string so it appears nowhere in s.
    s0 = p.b
    n = 5
    while s0.find('#' + ('!' * n)) > -1:
        n += 1
    comment = '#' + ('!' * n)
    # Create a dict of directives.
    d = {}
    for z in g.globalDirectiveList:
        d[z] = True
    # Convert all Leonine lines to special comments.
    i, lines, result = 0, g.splitLines(s0), []
    while i < len(lines):
        progress = i
        s = lines[i]
        # Comment out any containing a section reference.
        j = s.find('<<')
        k = s.find('>>') if j > -1 else -1
        if -1 < j < k:
            result.append(comment + s)
            # Generate a properly-indented pass line.
            j2 = g.skip_ws(s, 0)
            result.append('%spass\n' % (' ' * j2))
        elif s.lstrip().startswith('@'):
            # Comment out all other Leonine constructs.
            if starts_doc_part(s):
                # Comment the entire doc part, until @c or @code.
                result.append(comment + s)
                i += 1
                while i < len(lines):
                    s = lines[i]
                    result.append(comment + s)
                    i += 1
                    if ends_doc_part(s):
                        break
            else:
                j = g.skip_ws(s, 0)
                assert s[j] == '@'
                j += 1
                k = g.skip_id(s, j, chars='-')
                if k > j:
                    word = s[j: k]
                    if word == 'others':
                        # Remember the original @others line.
                        result.append(comment + s)
                        # Generate a properly-indented pass line.
                        result.append('%spass\n' % (' ' * (j - 1)))
                    else:
                        # Comment only Leo directives, not decorators.
                        result.append(comment + s if word in d else s)
                else:
                    result.append(s)
        else:
            # A plain line.
            result.append(s)
        if i == progress:
            i += 1
    return comment, ''.join(result)
#@+node:ekr.20150531042830.1: *5* starts_doc_part & ends_doc_part
def starts_doc_part(s):
    '''Return True if s word matches @ or @doc.'''
    for delim in ('@\n', '@doc\n', '@ ', '@doc '):
        if s.startswith(delim):
            return True
    return False

def ends_doc_part(s):
    '''Return True if s word matches @c or @code.'''
    for delim in ('@c\n', '@code\n', '@c ', '@code '):
        if s.startswith(delim):
            return True
    return False
#@+node:ekr.20150529095117.1: *5* uncomment_leo_lines
def uncomment_leo_lines(comment, p, s0):
    '''Reverse the effect of comment_leo_lines.'''
    lines = g.splitLines(s0)
    i, result = 0, []
    while i < len(lines):
        progress = i
        s = lines[i]
        i += 1
        if s.find(comment) == -1:
            # A regular line.
            result.append(s)
        else:
            # One or more special lines.
            i = uncomment_special_lines(comment, i, lines, p, result, s)
        assert progress < i
    return ''.join(result).rstrip() + '\n'
#@+node:ekr.20150531041720.1: *5* uncomment_special_line & helpers
def uncomment_special_lines(comment, i, lines, p, result, s):
    '''
    s is a line containing the comment delim.
    i points at the *next* line.
    Handle one or more lines, appending stripped lines to result.
    '''
    s = s.lstrip().lstrip(comment)
    if starts_doc_part(s):
        result.append(s)
        while i < len(lines):
            s = lines[i].lstrip().lstrip(comment)
            i += 1
            result.append(s)
            if ends_doc_part(s):
                break
        return i
    j = s.find('<<')
    k = s.find('>>') if j > -1 else -1
    if -1 < j < k or s.find('@others') > -1:
        # A section reference line or an @others line.
        # Such lines are followed by a pass line.
        # The beautifier may insert blank lines before the pass line.
        kind = 'section ref' if -1 < j < k else '@others'
        # Restore the original line, including leading whitespace.
        result.append(s)
        # Skip blank lines.
        while i < len(lines) and not lines[i].strip():
            i += 1
        # Skip the pass line.
        if i < len(lines) and lines[i].lstrip().startswith('pass'):
            i += 1
        else:
            g.trace('*** no pass after %s: %s' % (kind, p.h))
    else:
        # A directive line.
        result.append(s)
    return i
#@+node:ekr.20180328065332.1: *3* Check conventions stuff
#@+node:ekr.20171208042251.1: *4* @@button check-conventions (no longer used)
g.cls()
if c.changed: c.save()

import imp
import leo.core.leoCheck as leoCheck
imp.reload(leoCheck)

do_all = True
do_string = True

fails = []
    # All of Leo's core files pass!
fn = g.os_path_finalize_join(g.app.loadDir, '..', 'core', 'leoTest.py')
<< define s >>
<< old tests >>
if do_all:
    utils = leoCheck.ProjectUtils()
    aList = utils.project_files('leo', force_all=False)
    # g.printList(aList)
    for fn in aList:
        sfn = g.shortFileName(fn)
        if sfn in fails:
            print('===== skipping', sfn)
        else:
            print('==== fn', sfn)
            leoCheck.ConventionChecker(c).check(fn=fn)
elif do_string: # Test string s.
    leoCheck.ConventionChecker(c).check(s=s)
else: # Test an actual file.
    leoCheck.ConventionChecker(c).check(fn=fn)
#@+node:ekr.20171208105236.1: *5* << define s >>
s = '''\
class T:
    
    def __init__(self, tempNode):
        self.tempNode = tempNode.copy()
    
    def setUp(self):
        tempNode = self.tempNode
        while tempNode.firstChild():
            tempNode.firstChild().doDelete()
'''

s_ok2 = '''
class Context(object):
    def __init__ (self, parent_context):
        self.parent_context = parent_context
        if parent_context:
            parent_context.inner_contexts_list.append(self)
'''

s_ok= '''
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
'''

#@+node:ekr.20171210062719.1: *5* << old tests >>
s_passes_1 = '''\
class C1:
        
    def f1(self, p):
        print(p.v)
        
    def f2(self, p):
        self.f1(p.v) # WRONG

'''


s_1 = '''\
class C1:

    def __init__(self, c):
        self.c = c
        c.theTagController = self
        
    def add_tag(self, p):
        pass

class C2:

    def oops(self, p):
        c.tagController.add_tag(p.v,tag)
            # WRONG: should be p.

'''


s_2 = '''\
class TagController:

    def __init__(self, c):
        self.c = c
        c.theTagController = self

    def add_tag(self, p, tag):
        # Will fail if p is a vnode
        tags = set(p.v.u.get('__node_tags', set([])))

class LeoTagWidget(QtWidgets.QWidget):

    def __init__(self,c,parent=None):
        self.c = c
        self.tc = self.c.theTagController

    def add_tag(self, event=None):
        p = self.c.p
        self.tc.add_tag(p.v,tag) # WRONG: should be p.

'''
#@+node:ekr.20160109150703.1: *4* class Stats (old & stupid, from )
class Stats(object):
    '''A class containing global statistics & other data'''
    @others
#@+node:ekr.20160109150703.2: *5*  sd.ctor
def __init__ (self):

    # Files...
    # self.completed_files = [] # Files handled by do_files.
    # self.failed_files = [] # Files that could not be opened.
    # self.files_list = [] # Files given by user or by import statements.
    # self.module_names = [] # Module names corresponding to file names.

    # Contexts.
    # self.context_list = {}
        # Keys are fully qualified context names; values are contexts.
    # self.modules_dict = {}
        # Keys are full file names; values are ModuleContext's.

    # Statistics...
    # self.n_chains = 0
    self.n_contexts = 0
    # self.n_errors = 0
    self.n_lambdas = 0
    self.n_modules = 0
    # self.n_relinked_pointers = 0
    # self.n_resolvable_names = 0
    # self.n_resolved_contexts = 0
    # self.n_relinked_names = 0

    # Names...
    self.n_attributes = 0
    self.n_expressions = 0
    self.n_ivars = 0
    self.n_names = 0        # Number of symbol table entries.
    self.n_del_names = 0
    self.n_load_names = 0
    self.n_param_names = 0
    self.n_param_refs = 0
    self.n_store_names = 0

    # Statements...
    self.n_assignments = 0
    self.n_calls = 0
    self.n_classes = 0
    self.n_defs = 0
    self.n_fors = 0
    self.n_globals = 0
    self.n_imports = 0
    self.n_lambdas = 0
    self.n_list_comps = 0
    self.n_returns = 0
    self.n_withs = 0

    # Times...
    self.parse_time = 0.0
    self.pass1_time = 0.0
    self.pass2_time = 0.0
    self.total_time = 0.0
#@+node:ekr.20160109150703.6: *5* sd.print_times
def print_times (self):

    sd = self
    times = (
        'parse_time',
        'pass1_time',
        # 'pass2_time', # the resolve_names pass is no longer used.
        'total_time',
    )
    max_n = 5
    for s in times:
        max_n = max(max_n,len(s))
    print('\nScan times...\n')
    for s in times:
        pad = ' ' * (max_n - len(s))
        print('%s%s: %2.2f' % (pad,s,getattr(sd,s)))
    print('')
#@+node:ekr.20160109150703.7: *5* sd.print_stats
def print_stats (self):

    sd = self
    table = (
        '*', 'errors',

        '*Contexts',
        'classes','contexts','defs','modules',

        '*Statements',
        'assignments','calls','fors','globals','imports',
        'lambdas','list_comps','returns','withs',

        '*Names',
        'attributes','del_names','load_names','names',
        'param_names','param_refs','store_names',
        #'resolvable_names','relinked_names','relinked_pointers',
        # 'ivars',
        # 'resolved_contexts',
    )
    max_n = 5
    for s in table:
        max_n = max(max_n,len(s))
    print('\nStatistics...\n')
    for s in table:
        var = 'n_%s' % s
        pad = ' ' * (max_n - len(s))
        if s.startswith('*'):
            if s[1:].strip():
                print('\n%s\n' % s[1:])
            else:
                pass # print('')
        else:
            pad = ' ' * (max_n - len(s))
            print('%s%s: %s' % (pad,s,getattr(sd,var)))
    print('')
#@+node:ekr.20171211054600.1: *4* OLD checkConventions (leoCheck.py)
def checkConventions(c):
    '''
    A stand-alone version of the @button node that tested the
    ConventionChecker class.
    
    The check-conventions command in checkerCommands.py saves c and reloads
    the leoCheck module before calling this function.
    '''
    g.cls()
    kind = 'all'
    project_name = 'leo'  # 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'
    assert kind in ('all', 'file', 'production', 'string'), repr(kind)
    fn = g.os_path_finalize_join(g.app.loadDir, '..', 'plugins', 'qt_tree.py')
    report_stats = True # and kind != 'production'
    trace_fn = True
    trace_skipped = False
    fails_dict = {
        'coverage': ['cmdline.py',],
        'lib2to3': ['fixer_util.py', 'fix_dict.py', 'patcomp.py', 'refactor.py'],
        'leo': [], # All of Leo's core files pass.
        'pylint': [
            'base.py', 'classes.py', 'format.py',
            'logging.py', 'python3.py', 'stdlib.py', 
            'docparams.py', 'lint.py',
        ],
        'rope': ['objectinfo.py', 'objectdb.py', 'runmod.py',],
    }
    fails = fails_dict.get(project_name, [])
    << define s >>
    s = g.adjustTripleString(s, c.tab_width)
    << old tests >>
    stats = Stats()
    if kind == 'production':
        for p in g.findRootsWithPredicate(c, c.p, predicate=None):
            x = ConventionChecker(c, stats)
            x.check(fn=g.fullPath(c, p), trace_fn=trace_fn)
    elif kind == 'all':
        utils = ProjectUtils()
        aList = utils.project_files(project_name, force_all=False)
        if aList:
            t1 = time.clock()
            for fn in aList:
                sfn = g.shortFileName(fn)
                if sfn in fails or fn in fails:
                    if trace_skipped: print('===== skipping', sfn)
                else:
                    ConventionChecker(c, stats).check(fn=fn, trace_fn=trace_fn)
            t2 = time.clock()
            print('%s files in %4.2f sec.' % (len(aList), (t2-t1)))
        else:
            print('no files for project: %s' % (project_name))
    elif kind == 'string':
        ConventionChecker(c, stats).check(s=s)
    else:
        assert kind == 'file', repr(kind)
        ConventionChecker(c, stats).check(fn=fn)
    if report_stats:
        stats.report()
#@+node:ekr.20171211054736.2: *5* << define s >>
s = '''\
class T:
    
    def __init__(self, tempNode):
        self.tempNode = tempNode.copy()
    
    def setUp(self):
        tempNode = self.tempNode
        while tempNode.firstChild():
            tempNode.firstChild().doDelete()
'''

s_ok2 = '''
class Context(object):
    def __init__ (self, parent_context):
        self.parent_context = parent_context
        if parent_context:
            parent_context.inner_contexts_list.append(self)
'''
assert s_ok2

s_ok= '''
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
'''
assert s_ok

#@+node:ekr.20171211054736.3: *5* << old tests >>
s_passes_1 = '''\
class C1:
        
    def f1(self, p):
        print(p.v)
        
    def f2(self, p):
        self.f1(p.v) # WRONG

'''
assert s_passes_1

s_1 = '''\
class C1:

    def __init__(self, c):
        self.c = c
        c.theTagController = self
        
    def add_tag(self, p):
        pass

class C2:

    def oops(self, p):
        c.tagController.add_tag(p.v,tag)
            # WRONG: should be p.

'''
assert s_1


s_2 = '''\
class TagController:

    def __init__(self, c):
        self.c = c
        c.theTagController = self

    def add_tag(self, p, tag):
        # Will fail if p is a vnode
        tags = set(p.v.u.get('__node_tags', set([])))

class LeoTagWidget(QtWidgets.QWidget):

    def __init__(self,c,parent=None):
        self.c = c
        self.tc = self.c.theTagController

    def add_tag(self, event=None):
        p = self.c.p
        self.tc.add_tag(p.v,tag) # WRONG: should be p.

'''
assert s_2
#@+node:ekr.20180813063846.1: *3* Fast-draw branches
#@+node:ekr.20180808075509.1: *4* qtree.partialDraw & helpers (never used)
def partialDraw(self, p):

    trace = True and not g.unitTesting
    c = self.c
    if 1:
        self.drawVisible()
    else:
        first_p = c.hoistStack[-1].p if c.hoistStack else c.rootPosition()
        aList1 = self.countVisible(first_p=first_p, target_p=p)
        aList2 = self.countVisible(first_p=p, target_p=None)
        if trace:
            n1, n2 = len(aList1), len(aList2)
            g.trace('%s + %s = %s' % (n1, n2, n1+n2))
        if 1:
            # Draw everything.
            self.drawList(aList1 + aList2)
        else:
            aList = self.computeVisiblePositions(aList1, aList2, p)
            self.drawList(aList)
#@+node:ekr.20180809110957.1: *5* qtree.computeVisiblePositions
def computeVisiblePositions(self, aList1, aList2, p):
    '''
    Compute the list of *visible* positions to be drawn.
    
    This is tricky.  We don't want to scroll the screen unnecessarily.
    '''
    # Show everything if possible.
    if len(aList1) + len(aList2) <= self.size:
        return aList1 + aList2
    c = self.c
    while p.hasParent():
        p.moveToParent()
    aList = []
    for i in range(self.size):
        aList.append(p.copy())
        p.moveToVisNext(c)
        if not p:
            break
    return aList
#@+node:ekr.20180809123937.1: *5* qtree.countVisible
def countVisible(self, first_p, target_p):
    """
    Return the number of visible positions from first_p to target_p.
    """
    c = self.c
    aList, p = [first_p.copy()], first_p.copy()
    while p:
        if p == target_p:
            return aList[:-1]
        v = p.v
        # if v.isExpanded() and v.hasChildren():
        if (v.statusBits & v.expandedBit) != 0 and v.children:
            # p.moveToFirstChild()
            p.stack.append((v, p._childIndex),)
            p.v = v.children[0]
            p._childIndex = 0
            aList.append(p.copy())
            continue
        # if p.hasNext():
        parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
        if p._childIndex + 1 < len(parent_v.children):
            # p.moveToNext()
            p._childIndex += 1
            p.v = parent_v.children[p._childIndex]
            aList.append(p.copy())
            continue
        #
        # A fast version of p.moveToThreadNext().
        # We look for a parent with a following sibling.
        while p.stack:
            # p.moveToParent()
            p.v, p._childIndex = p.stack.pop()
            # if p.hasNext():
            parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
            if p._childIndex + 1 < len(parent_v.children):
                # p.moveToNext()
                p._childIndex += 1
                p.v = parent_v.children[p._childIndex]
                break # Found: moveToThreadNext()
        else:
            break # Not found.
        # Found moveToThreadNext()
        aList.append(p.copy())
        continue
    if target_p:
        g.trace('NOT FOUND:', target_p.h)
    return aList
#@+node:ekr.20180809115019.1: *5* qtree.drawList
def drawList(self, aList):
    
    trace = False
    c = self.c
    parents = []
    # Clear the widget.
    w = self.treeWidget
    w.clear()
    # Clear the dicts.
    self.initData()
    for p in aList:
        level = p.level()
        parent_item = w if level == 0 else parents[level-1]
        item = QtWidgets.QTreeWidgetItem(parent_item)
        item.setFlags(item.flags() | QtCore.Qt.ItemIsEditable)
        item.setChildIndicatorPolicy(
            item.ShowIndicator if p.hasChildren()
            else item.DontShowIndicator)
        item.setExpanded(bool(p.hasChildren() and p.isExpanded()))
        self.items.append(item)
        if trace:
            print('')
            g.trace('===== level', level, p.h)
            g.trace('parent', id(parent_item), parent_item.__class__.__name__)
            g.trace('item  ', id(item), item.__class__.__name__)
            self.print_parents(parents, 1)
        # Update parents.
        if level == 0:
            parents = []
        else:
            parents = parents[:level]
        parents.append(item)
        if trace:
            self.print_parents(parents, 2)
        # Update the dicts.  Like rememberItem.
        itemHash = self.itemHash(item)
        self.item2positionDict[itemHash] = p.copy()
        self.item2vnodeDict[itemHash] = p.v
        self.position2itemDict[p.key()] = item
        d = self.vnode2itemsDict
        v = p.v
        aList = d.get(v, [])
        aList.append(item)
        d[v] = aList
        # Enter the headline.
        item.setText(0, p.h)
        # Set current item.
        if p == c.p:
            w.setCurrentItem(item)
#@+node:ekr.20180809111725.1: *5* qtree.printParents
def print_parents(self, parents, tag):
    print(tag)
    g.printObj([
        '%10s %s' % (id(z), z.__class__.__name__)
            for z in parents])
#@+node:ekr.20180810060655.1: *4* test vieldVisible
g.cls()
import time
tree = c.frame.tree
if 1: # works
    for p in c.all_positions():
        p.expand()
elif 1:
    for p in c.all_positions():
        p.v.expandedPositions = []
    for p in c.all_positions():
        p.v.expand()
        p.v.expandedPositions.append(p.copy())
else: # Doesn't work
    for v in c.all_nodes():
        v.expand()
    c.redraw() # This would be wrong.
if 1:
    t1 = time.clock()
    for i in range(1):
        aList1 = [z.copy() for z in tree.slowYieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('slow: %6.3f' % (t2-t1))
if 1:
    t1 = time.clock()
    for i in range(1):
        aList2 = [z.copy() for z in tree.yieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('fast: %6.3f' % (t2-t1))
v1 = [z.v for z in aList1]
v2 = [z.v for z in aList2]
try:
    i = 0
    while i < min(len(aList1), len(aList2)) and aList1[i] == aList2[i]:
        # print(i, aList1[i].h)
        i += 1
    if i == len(aList1) == len(aList2):
        print('OK')
    else:
        print('FAIL', i, len(aList1), len(aList2))
    assert aList1 == aList2, (len(aList1), len(aList2))
    assert v1 == v2, (len(v1), len(v2))
finally:
    for v in c.all_nodes():
        v.contract()
    c.redraw()
#@+node:ekr.20180810111515.1: *4* benchmark
import time
t1 = time.clock()
c.expandAllHeadlines()
t2 = time.clock()
print('expand: %5.2f sec' % (t2-t1))
t3 = time.clock()
tree = c.frame.tree
w = tree.treeWidget
n = 0
for p in tree.yieldVisible(c.rootPosition()):
    n += 1
    # c.selectPosition(p)
    item = tree.position2itemDict.get(c.p.key())
    if item:
        w.setCurrentItem(item)
t4 = time.clock()
print('%s nodes in %5.2f sec' % (n, t4-t3))
#@+node:ekr.20110605121601.17879: *4* qtree.rememberItem
def rememberItem(self, p, item):

    v = p.v
    # Update position dicts.
    itemHash = self.itemHash(item)
    self.position2itemDict[p.key()] = item
    self.item2positionDict[itemHash] = p.copy() # was item
    # Update item2vnodeDict.
    self.item2vnodeDict[itemHash] = v # was item
    # Update vnode2itemsDict.
    d = self.vnode2itemsDict
    aList = d.get(v, [])
    if item in aList:
        g.trace('*** ERROR *** item already in list: %s, %s' % (item, aList))
    else:
        aList.append(item)
    d[v] = aList
#@+node:ekr.20120219154958.10488: *4* LM.initFocusAndDraw (not used)
def initFocusAndDraw(self, c, fileName):

    def init_focus_handler(timer, c=c, p=c.p):
        '''Idle-time handler for initFocusAndDraw'''
        c.initialFocusHelper()
        c.outerUpdate()
        timer.stop()

    # This must happen after the code in getLeoFile.
    timer = g.IdleTime(init_focus_handler, delay=0.1, tag='getLeoFile')
    if timer:
        timer.start()
    else:
        # Default code.
        c.selectPosition(c.p)
        c.initialFocusHelper()
        c.k.showStateAndMode()
        c.outerUpdate()
#@+node:ekr.20150312225028.29: *3* leoViews project
This was a major project, now abandoned.
#@+node:ekr.20150312225028.31: *4* class OrganizerData
class OrganizerData:
    '''A class containing all data for a particular organizer node.'''
    def __init__ (self,h,unl,unls):
        self.anchor = None # The anchor position of this od node.
        self.children = [] # The direct child od nodes of this od node.
        self.closed = False # True: this od node no longer accepts new child od nodes.
        self.drop = True # Drop the unl for this od node when associating positions with unls.
        self.descendants = None # The descendant od nodes of this od node.
        self.exists = False # True: this od was created by @existing-organizer:
        self.h = h # The headline of this od node.
        self.moved = False # True: the od node has been moved to a global move list.
        self.opened = False # True: the od node has been opened.
        self.organized_nodes = [] # The list of positions organized by this od node.
        self.parent_od = None # The parent od node of this od node. (None is valid.)
        self.p = None # The position of this od node.
        self.parent = None # The original parent position of all nodes organized by this od node.
            # If parent_od is None, this will be the parent position of the od node.
        self.source_unl = None # The unl of self.parent.
        self.unl = unl # The unl of this od node.
        self.unls = unls # The unls contained in this od node.
        self.visited = False # True: demote_helper has already handled this od node.
    def __repr__(self):
        return 'OrganizerData: %s' % (self.h or '<no headline>')
    __str__ = __repr__
#@+node:ekr.20150312225028.32: *4* class ViewController
class ViewController:
    << docstring >>
    @others
#@+node:ekr.20150312225028.33: *5*  << docstring >> (class ViewController)
'''
A class to handle @views trees and related operations.
Such trees have the following structure:

- @views
  - @auto-view <unl of @auto node>
    - @organizers
      - @organizer <headline>
    - @clones
    
The body text of @organizer and @clones consists of unl's, one per line.
'''
#@+node:ekr.20150312225028.34: *5*  vc.ctor & vc.init
def __init__ (self,c):
    '''Ctor for ViewController class.'''
    self.c = c
    self.headline_ivar = '_imported_headline'
    self.init()
    
def init(self):
    '''
    Init all ivars of this class.
    Unit tests may call this method to ensure that this class is re-inited properly.
    '''
    self.all_ods = []
        # List of all od nodes.
    self.anchors_d = {}
        # Keys are anchoring positions, values are sorted lists of ods.
    self.anchor_offset_d = {}
        # Keys are anchoring positions, values are ints.
    self.existing_ods = []
        # List of od instances corresponding to @existing-organizer: nodes.
    self.global_bare_organizer_node_list = []
        # List of organizers that have no parent organizer node.
        # This list excludes existing organizer nodes.
    self.headlines_dict = {}
        # Keys are vnodes; values are list of child headlines.
    self.imported_organizers_list = []
        # The list of nodes that have children on entry, such as class nodes.
    self.n_nodes_scanned = 0
        # Number of nodes scanned by demote.
    self.organizer_ods = []
        # List of od instances corresponding to @organizer: nodes.
    self.organizer_unls = []
        # The list of od.unl for all od instances in self.organizer_ods.
    self.root = None
        # The position of the @auto node.
    self.pending = []
        # The list of nodes pending to be added to an organizer.
    self.stack = []
        # The stack containing real and virtual parent nodes during the main loop.
    self.temp_node = None
        # The parent position of all holding cells.
    self.trail_write_1 = None
        # The trial write on entry.
    self.views_node = None
        # The position of the @views node.
    self.work_list = []
        # A gloal list of (parent,child) tuples for all nodes that are
        # to be moved to **non-existing** organizer nodes.
        # **Important**: Nodes are moved in the order they appear in this list:
        # the tuples contain no childIndex component!
        # This list is the "backbone" of this class:
        # - The front end (demote and its helpers) adds items to this list.
        # - The back end (move_nodes and its helpers) moves nodes using this list.
#@+node:ekr.20150312225028.35: *5* vc.Entry points
#@+node:ekr.20150312225028.36: *6* vc.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self,root):
    # Define class ConvertController.
    @others
    vc = self
    c = vc.c
    if root.isAtFileNode():
        ConvertController(c,root).run()
    else:
        g.es_print('not an @file node:',root.h)
#@+node:ekr.20150312225028.37: *7* class ConvertController
class ConvertController:
    def __init__ (self,c,p):
        self.c = c
        # self.ic = c.importCommands
        self.vc = c.viewController
        self.root = p.copy()
    @others
#@+node:ekr.20150312225028.38: *8* cc.delete_at_auto_view_nodes
def delete_at_auto_view_nodes(self,root):
    '''Delete all @auto-view nodes pertaining to root.'''
    cc = self
    vc = cc.vc
    while True:
        p = vc.has_at_auto_view_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20150312225028.39: *8* cc.import_from_string
def import_from_string(self,s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    scanner = ic.scanner_for_ext(ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True,parent=p,s=s)
    p.h = root.h.replace('@file','@auto' if ok else '@@auto')
    return ok,p
#@+node:ekr.20150312225028.40: *8* cc.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root,vc = cc.root,c.viewController
    # set the headline_ivar for all vnodes.
    t1 = time.clock()
    cc.set_expected_imported_headlines(root)
    t2 = time.clock()
    # Delete all previous @auto-view nodes for this tree.
    cc.delete_at_auto_view_nodes(root)
    t3 = time.clock()
    # Ensure that all nodes of the tree are regularized.
    ok = vc.prepass(root)
    t4 = time.clock()
    if not ok:
        g.es_print('Can not convert',root.h,color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
            # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4-t3),
            '\n  total:                           %4.2f sec' % (t4-t1))
        return
    # Create the appropriate @auto-view node.
    at_auto_view = vc.update_before_write_at_auto_file(root)
    t5 = time.clock()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.clock()
    if trace and trace_s:
        g.trace('source file...\n',s)
    # Import the @auto string.
    ok,p = cc.import_from_string(s)
    t7 = time.clock()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = vc.at_auto_view_body(p)
        # Recreate the organizer nodes, headlines, etc.
        ok = vc.update_after_read_at_auto_file(p)
        t8 = time.clock()
        if not ok:
            p.h = '@@' + p.h
            g.trace('restoring original @auto file')
            ok,p = cc.import_from_string(s)
            if ok:
                p.h = '@@' + p.h + ' (restored)'
                if p.next():
                    p.moveAfter(p.next())
        t9 = time.clock()
    else:
        t8 = t9 = time.clock()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
        # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4-t3),
        '\n  update_before_write_at_auto_file:%4.2f sec' % (t5-t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6-t5),
        '\n  import_from_string:              %4.2f sec' % (t7-t6),
        '\n  update_after_read_at_auto_file   %4.2f sec' % (t8-t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9-t8),
        '\n  total:                           %4.2f sec' % (t9-t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20150312225028.41: *8* cc.set_expected_imported_headlines
def set_expected_imported_headlines(self,root):
    '''Set the headline_ivar for all vnodes.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    aClass = ic.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic,atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c,root,importing=True)
    fn = c.os_path_finalize_join(theDir,root.h)
    fn = root.h.replace('\\','/')
    junk,fn = g.os_path_split(fn)
    fn,junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner,'headlineForNode'):
        ivar = cc.vc.headline_ivar
        for p in root.subtree():
            if not hasattr(p.v,ivar):
                h = scanner.headlineForNode(fn,p)
                setattr(p.v,ivar,h)
                if trace and h != p.h:
                    g.trace('==>',h) # p.h,'==>',h
#@+node:ekr.20150312225028.42: *8* cc.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind = '@file',
        nosentinels = True,
        perfectImportFlag = False,
        scriptWrite = False,
        thinFile = True,
        toString = True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:',ok,'s:...\n'+s)
    return s
#@+node:ekr.20150312225028.43: *6* vc.pack & helper
def pack(self):
    '''
    Undoably convert c.p to a packed @view node, replacing all cloned
    children of c.p by unl lines in c.p.b.
    '''
    vc = self
    c,u = vc.c,vc.c.undoer
    vc.init()
    changed = False
    root = c.p
    # Create an undo group to handle changes to root and @views nodes.
    # Important: creating the @views node does *not* invalidate any positions.'''
    u.beforeChangeGroup(root,'view-pack')
    if not vc.has_at_views_node():
        changed = True
        bunch = u.beforeInsertNode(c.rootPosition())
        views = vc.find_at_views_node()
            # Creates the @views node as the *last* top-level node
            # so that no positions become invalid as a result.
        u.afterInsertNode(views,'create-views-node',bunch)
    # Prepend @view if need.
    if not root.h.strip().startswith('@'):
        changed = True
        bunch = u.beforeChangeNodeContents(root)
        root.h = '@view ' + root.h.strip()
        u.afterChangeNodeContents(root,'view-pack-update-headline',bunch)
    # Create an @view node as a clone of the @views node.
    bunch = u.beforeInsertNode(c.rootPosition())
    new_clone = vc.create_view_node(root)
    if new_clone:
        changed = True
        u.afterInsertNode(new_clone,'create-view-node',bunch)
    # Create a list of clones that have a representative node
    # outside of the root's tree.
    reps = [vc.find_representative_node(root,p)
        for p in root.children()
            if vc.is_cloned_outside_parent_tree(p)]
    reps = [z for z in reps if z is not None]
    if reps:
        changed = True
        bunch = u.beforeChangeTree(root)
        c.setChanged(True)
        # Prepend a unl: line for each cloned child.
        unls = ['unl: %s\n' % (vc.unl(p)) for p in reps]
        root.b = ''.join(unls) + root.b
        # Delete all child clones in the reps list.
        v_reps = set([p.v for p in reps])
        while True:
            for child in root.children():
                if child.v in v_reps:
                    child.doDelete()
                    break
            else: break
        u.afterChangeTree(root,'view-pack-tree',bunch)
    if changed:
        u.afterChangeGroup(root,'view-pack')
        c.selectPosition(root)
        c.redraw()
#@+node:ekr.20150312225028.44: *7* vc.create_view_node
def create_view_node(self,root):
    '''
    Create a clone of root as a child of the @views node.
    Return the *newly* cloned node, or None if it already exists.
    '''
    vc = self
    c = vc.c
    # Create a cloned child of the @views node if it doesn't exist.
    views = vc.find_at_views_node()
    for p in views.children():
        if p.v == c.p.v:
            return None
    p = root.clone()
    p.moveToLastChildOf(views)
    return p
#@+node:ekr.20150312225028.45: *6* vc.unpack
def unpack(self):
    '''
    Undoably unpack nodes corresponding to leading unl lines in c.p to child clones.
    Return True if the outline has, in fact, been changed.
    '''
    vc = self
    c,root,u = vc.c,vc.c.p,vc.c.undoer
    vc.init()
    # Find the leading unl: lines.
    i,lines,tag = 0,g.splitLines(root.b),'unl:'
    for s in lines:
        if s.startswith(tag): i += 1
        else: break
    changed = i > 0
    if changed:
        bunch = u.beforeChangeTree(root)
        # Restore the body
        root.b = ''.join(lines[i:])
        # Create clones for each unique unl.
        unls = list(set([s[len(tag):].strip() for s in lines[:i]]))
        for unl in unls:
            p = vc.find_absolute_unl_node(unl)
            if p: p.clone().moveToLastChildOf(root)
            else: g.trace('not found: %s' % (unl))
        c.setChanged(True)
        c.undoer.afterChangeTree(root,'view-unpack',bunch)
        c.redraw()
    return changed
#@+node:ekr.20150312225028.46: *6* vc.update_before_write_at_auto_file
def update_before_write_at_auto_file(self,root):
    '''
    Update the @auto-view node for root, an @auto node. Create @organizer,
    @existing-organizer, @clones and @headlines nodes as needed.
    This *must not* be called for trial writes.
    '''
    trace = False and not g.unitTesting
    vc = self
    c = vc.c
    changed = False
    t1 = time.clock()
    # Create lists of cloned and organizer nodes.
    clones,existing_organizers,organizers = \
        vc.find_special_nodes(root)
    # Delete all children of the @auto-view node for this @auto node.
    at_auto_view = vc.find_at_auto_view_node(root)
    if at_auto_view.hasChildren():
        changed = True
        at_auto_view.deleteAllChildren()
    # Create the single @clones node.
    if clones:
        at_clones = vc.find_at_clones_node(root)
        at_clones.b = ''.join(
            ['gnx: %s\nunl: %s\n' % (z[0],z[1]) for z in clones])
    # Create the single @organizers node.
    if organizers or existing_organizers:
        at_organizers = vc.find_at_organizers_node(root)
    # Create one @organizers: node for each organizer node.
    for p in organizers:
        # g.trace('organizer',p.h)
        at_organizer = at_organizers.insertAsLastChild()
        at_organizer.h = '@organizer: %s' % p.h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create one @existing-organizer node for each existing organizer.
    ivar = vc.headline_ivar
    for p in existing_organizers:
        at_organizer = at_organizers.insertAsLastChild()
        h = getattr(p.v,ivar,p.h)
        if trace and h != p.h: g.trace('==>',h) # p.h,'==>',h
        at_organizer.h = '@existing-organizer: %s' % h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create the single @headlines node.
    vc.create_at_headlines(root)
    if changed and not g.unitTesting:
        g.es_print('updated @views node in %4.2f sec.' % (
            time.clock()-t1))
    if changed:
        c.redraw()
    return at_auto_view # For at-file-to-at-auto command.
#@+node:ekr.20150312225028.47: *7* vc.create_at_headlines
def create_at_headlines(self,root):
    '''Create the @headlines node for root, an @auto file.'''
    vc = self
    c = vc.c
    result = []
    ivar = vc.headline_ivar
    for p in root.subtree():
        h = getattr(p.v,ivar,None)
        if h is not None and p.h != h:
            # g.trace('custom:',p.h,'imported:',h)
            unl = vc.relative_unl(p,root)
            aList = unl.split('-->')
            aList[-1] = h
            unl = '-->'.join(aList)
            result.append('imported unl: %s\nhead: %s\n' % (
                unl,p.h))
            delattr(p.v,ivar)
    if result:
        p = vc.find_at_headlines_node(root)
        p.b = ''.join(result)
#@+node:ekr.20150312225028.48: *7* vc.find_special_nodes
def find_special_nodes(self,root):
    '''
    Scan root's tree, looking for organizer and cloned nodes.
    Exclude organizers on imported organizers list.
    '''
    trace = False and not g.unitTesting
    verbose = False
    vc = self
    clones,existing_organizers,organizers = [],[],[]
    if trace: g.trace('imported existing',
        [v.h for v in vc.imported_organizers_list])
    for p in root.subtree():
        if p.isCloned():
            rep = vc.find_representative_node(root,p)
            if rep:
                unl = vc.relative_unl(p,root)
                gnx = rep.v.gnx
                clones.append((gnx,unl),)
        if p.v in vc.imported_organizers_list:
            # The node had children created by the importer.
            if trace and verbose: g.trace('ignore imported existing',p.h)
        elif vc.is_organizer_node(p,root):
            # p.hasChildren and p.b is empty, except for comments.
            if trace and verbose: g.trace('organizer',p.h)
            organizers.append(p.copy())
        elif p.hasChildren():
            if trace and verbose: g.trace('existing',p.h)
            existing_organizers.append(p.copy())
    return clones,existing_organizers,organizers
#@+node:ekr.20150312225028.49: *6* vc.update_after_read_at_auto_file & helpers
def update_after_read_at_auto_file(self,root):
    '''
    Recreate all organizer nodes and clones for a single @auto node
    using the corresponding @organizer: and @clones nodes.
    '''
    trace = True and not g.unitTesting
    vc = self
    c = vc.c
    if not vc.is_at_auto_node(root):
        return # Not an error: it might be and @auto-rst node.
    old_changed = c.isChanged()
    try:
        vc.init()
        vc.root = root.copy()
        t1 = time.clock()
        vc.trial_write_1 = vc.trial_write(root)
        t2 = time.clock()
        at_organizers = vc.has_at_organizers_node(root)
        t3 = time.clock()
        if at_organizers:
            vc.create_organizer_nodes(at_organizers,root)
        t4 = time.clock()
        at_clones = vc.has_at_clones_node(root)
        if at_clones:
            vc.create_clone_links(at_clones,root)
        t5 = time.clock()
        n = len(vc.work_list)
        ok = vc.check(root)
        t6 = time.clock()
        if ok:
            vc.update_headlines_after_read(root)
        t7 = time.clock()
        c.setChanged(old_changed if ok else False)
            # To do: revert if not ok.
    except Exception:
        g.es_exception()
        n = 0
        ok = False
    if trace:
        if t7-t1 > 0.5: g.trace(
            '\n  trial_write:                 %4.2f sec' % (t2-t1),
            # '\n  has_at_organizers_node:    %4.2f sec' % (t3-t2),
            '\n  create_organizer_nodes:      %4.2f sec' % (t4-t3),
            '\n  create_clone_links:          %4.2f sec' % (t5-t4),
            '\n  check:                       %4.2f sec' % (t6-t5),
            '\n  update_headlines_after_read: %4.2f sec' % (t7-t6),
            '\n  total:                       %4.2f sec' % (t7-t1))
            # '\n  file:',root.h)
        # else: g.trace('total: %4.2f sec' % (t7-t1),root.h)
    if ok and n > 0:
        g.es('rearragned: %s' % (root.h),color='blue')
        g.es('moved %s nodes in %4.2f sec.' % (n,t7-t1))
        g.trace('@auto-view moved %s nodes in %4.2f sec. for' % (
            n,t2),root.h,noname=True)
    c.selectPosition(root)
    c.redraw()
    return ok
#@+node:ekr.20150312225028.50: *7* vc.check
def check (self,root):
    '''
    Compare a trial write or root with the vc.trail_write_1.
    Unlike the perfect-import checks done by the importer,
    we expecct an *exact* match, regardless of language.
    '''
    trace = True # and not g.unitTesting
    vc = self
    trial1 = vc.trial_write_1
    trial2 = vc.trial_write(root)
    if trial1 != trial2:
        g.pr('') # Don't use print: it does not appear with the traces.
        g.es_print('perfect import check failed for:',color='red')
        g.es_print(root.h,color='red')
        if trace:
            vc.compare_trial_writes(trial1,trial2)
            g.pr('')
    return trial1 == trial2
#@+node:ekr.20150312225028.51: *7* vc.create_clone_link
def create_clone_link(self,gnx,root,unl):
    '''
    Replace the node in the @auto tree with the given unl by a
    clone of the node outside the @auto tree with the given gnx.
    '''
    trace = False and not g.unitTesting
    vc = self
    p1 = vc.find_position_for_relative_unl(root,unl)
    p2 = vc.find_gnx_node(gnx)
    if p1 and p2:
        if trace: g.trace('relink',gnx,p2.h,'->',p1.h)
        if p1.b == p2.b:
            p2._relinkAsCloneOf(p1)
            return True
        else:
            g.es('body text mismatch in relinked node',p1.h)
            return False
    else:
        if trace: g.trace('relink failed',gnx,root.h,unl)
        return False
#@+node:ekr.20150312225028.52: *7* vc.create_clone_links
def create_clone_links(self,at_clones,root):
    '''
    Recreate clone links from an @clones node.
    @clones nodes contain pairs of lines (gnx,unl)
    '''
    vc = self
    lines = g.splitLines(at_clones.b)
    gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')]
    unls = [s[4:].strip() for s in lines if s.startswith('unl:')]
    # g.trace('at_clones.b',at_clones.b)
    if len(gnxs) == len(unls):
        vc.headlines_dict = {} # May be out of date.
        ok = True
        for gnx,unl in zip(gnxs,unls):
            ok = ok and vc.create_clone_link(gnx,root,unl)
        return ok
    else:
        g.trace('bad @clones contents',gnxs,unls)
        return False
#@+node:ekr.20150312225028.53: *7* vc.create_organizer_nodes & helpers
def create_organizer_nodes(self,at_organizers,root):
    '''
    root is an @auto node. Create an organizer node in root's tree for each
    child @organizer: node of the given @organizers node.
    '''
    vc = self
    c = vc.c
    trace = False and not g.unitTesting
    t1 = time.clock()
    vc.pre_move_comments(root)
        # Merge comment nodes with the next node.
    t2 = time.clock()
    vc.precompute_all_data(at_organizers,root)
        # Init all data required for reading.
    t3 = time.clock()
    vc.demote(root)
        # Traverse root's tree, adding nodes to vc.work_list.
    t4 = time.clock()
    vc.move_nodes()
        # Move nodes on vc.work_list to their final locations.
    t5 = time.clock()
    vc.post_move_comments(root)
        # Move merged comments to parent organizer nodes.
    t6 = time.clock()
    if trace: g.trace(
        '\n  pre_move_comments:   %4.2f sec' % (t2-t1),
        '\n  precompute_all_data: %4.2f sec' % (t3-t2),
        '\n  demote:              %4.2f sec' % (t4-t3),
        '\n  move_nodes:          %4.2f sec' % (t5-t4),
        '\n  post_move_comments:  %4.2f sec' % (t6-t5))
#@+node:ekr.20150312225028.54: *7* vc.update_headlines_after_read
def update_headlines_after_read(self,root):
    '''Handle custom headlines for all imported nodes.'''
    trace = False and not g.unitTesting
    vc = self
    # Remember the original imported headlines.
    ivar = vc.headline_ivar
    for p in root.subtree():
        if not hasattr(p.v,ivar):
            setattr(p.v,ivar,p.h)
    # Update headlines from @headlines nodes.
    at_headlines = vc.has_at_headlines_node(root)
    tag1,tag2 = 'imported unl: ','head: '
    n1,n2 = len(tag1),len(tag2)
    if at_headlines:
        lines = g.splitLines(at_headlines.b)
        unls  = [s[n1:].strip() for s in lines if s.startswith(tag1)]
        heads = [s[n2:].strip() for s in lines if s.startswith(tag2)]
    else:
        unls,heads = [],[]
    if len(unls) == len(heads):
        vc.headlines_dict = {} # May be out of date.
        for unl,head in zip(unls,heads):
            p = vc.find_position_for_relative_unl(root,unl)
            if p:
                if trace: g.trace('unl:',unl,p.h,'==>',head)
                p.h = head
    else:
        g.trace('bad @headlines body',at_headlines.b)
#@+node:ekr.20150312225028.55: *5* vc.Main Lines
#@+node:ekr.20150312225028.56: *6* vc.precompute_all_data & helpers
def precompute_all_data(self,at_organizers,root):
    '''Precompute all data needed to reorganize nodes.'''
    trace = False and not g.unitTesting
    vc = self
    t1 = time.clock() 
    vc.find_imported_organizer_nodes(root)
        # Put all nodes with children on vc.imported_organizer_node_list
    t2 = time.clock()
    vc.create_organizer_data(at_organizers,root)
        # Create OrganizerData objects for all @organizer:
        # and @existing-organizer: nodes.
    t3 = time.clock()
    vc.create_actual_organizer_nodes()
        # Create the organizer nodes in holding cells so positions remain valid.
    t4 = time.clock()
    vc.create_tree_structure(root)
        # Set od.parent_od, od.children & od.descendants for all ods.
    t5 = time.clock()
    vc.compute_all_organized_positions(root)
        # Compute the positions organized by each organizer.
        # ** Most of the time is spent here **.
    t6 = time.clock()
    vc.create_anchors_d()
        # Create the dictionary that associates positions with ods.
    t7 = time.clock()
    if trace: g.trace(
        '\n  find_imported_organizer_nodes:   %4.2f sec' % (t2-t1),
        '\n  create_organizer_data:           %4.2f sec' % (t3-t2),
        '\n  create_actual_organizer_nodes:   %4.2f sec' % (t4-t3),
        '\n  create_tree_structure:           %4.2f sec' % (t5-t4),
        '\n  compute_all_organized_positions: %4.2f sec' % (t6-t5),
        '\n  create_anchors_d:                %4.2f sec' % (t7-t6))
#@+node:ekr.20150312225028.57: *7* 1: vc.find_imported_organizer_nodes
def find_imported_organizer_nodes(self,root):
    '''
    Put the VNode of all imported nodes with children on
    vc.imported_organizers_list.
    '''
    trace = False # and not g.unitTesting
    vc = self
    aList = []
    for p in root.subtree():
        if p.hasChildren():
            aList.append(p.v)
    vc.imported_organizers_list = list(set(aList))
    if trace: g.trace([z.h for z in vc.imported_organizers_list])
#@+node:ekr.20150312225028.58: *7* 2: vc.create_organizer_data (od.p & od.parent)
def create_organizer_data(self,at_organizers,root):
    '''
    Create OrganizerData nodes for all @organizer: and @existing-organizer:
    nodes in the given @organizers node.
    '''
    vc = self
    vc.create_ods(at_organizers)
    vc.finish_create_organizers(root)
    vc.finish_create_existing_organizers(root)
    for od in vc.all_ods:
        assert od.parent,(od.exists,od.h)
#@+node:ekr.20150312225028.59: *8* vc.create_ods
def create_ods(self,at_organizers):
    '''Create all organizer nodes and the associated lists.'''
    # Important: we must completely reinit all data here.
    vc = self
    tag1 = '@organizer:'
    tag2 = '@existing-organizer:'
    vc.all_ods,vc.existing_ods,vc.organizer_ods = [],[],[]
    for at_organizer in at_organizers.children():
        h = at_organizer.h
        for tag in (tag1,tag2):
            if h.startswith(tag):
                unls = vc.get_at_organizer_unls(at_organizer)
                if unls:
                    organizer_unl = vc.drop_unl_tail(unls[0])
                    h = h[len(tag):].strip()
                    od = OrganizerData(h,organizer_unl,unls)
                    vc.all_ods.append(od)
                    if tag == tag1:
                        vc.organizer_ods.append(od)
                        vc.organizer_unls.append(organizer_unl)
                    else:
                        vc.existing_ods.append(od)
                        # Do *not* append organizer_unl to the unl list.
                else:
                    g.trace('===== no unls:',at_organizer.h)
#@+node:ekr.20150312225028.60: *8* vc.finish_create_organizers
def finish_create_organizers(self,root):
    '''Finish creating all organizers.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.organizer_ods[:]: 
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.parent = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.parent:
            od.anchor = od.parent
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This is, most likely, a true error.
            g.trace('===== removing od:',od.h)
            vc.organizer_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods
#@+node:ekr.20150312225028.61: *8* vc.finish_create_existing_organizers
def finish_create_existing_organizers(self,root):
    '''Finish creating existing organizer nodes.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.existing_ods[:]:
        od.exists = True
        assert od.unl not in vc.organizer_unls
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.p = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.p:
            od.anchor = od.p
            assert od.p.h == od.h,(od.p.h,od.h)  
            od.parent = od.p # Here, od.parent represents the "source" p.
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This arises when the imported node name doesn't match.
            g.trace('===== removing existing organizer:',od.h)
            vc.existing_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods

#@+node:ekr.20150312225028.62: *7* 3: vc.create_actual_organizer_nodes
def create_actual_organizer_nodes(self):
    '''
    Create all organizer nodes as children of holding cells. These holding
    cells ensure that moving an organizer node leaves all other positions
    unchanged.
    '''
    vc = self
    c = vc.c
    last = c.lastTopLevel()
    temp = vc.temp_node = last.insertAfter()
    temp.h = 'ViewController.temp_node'
    for od in vc.organizer_ods:
        holding_cell = temp.insertAsLastChild()
        holding_cell.h = 'holding cell for ' + od.h
        od.p = holding_cell.insertAsLastChild()
        od.p.h = od.h
#@+node:ekr.20150312225028.63: *7* 4: vc.create_tree_structure & helper
def create_tree_structure(self,root):
    '''Set od.parent_od, od.children & od.descendants for all ods.'''
    trace = False and not g.unitTesting
    vc = self
    # if trace: g.trace([z.h for z in data_list],g.callers())
    organizer_unls = [z.unl for z in vc.all_ods]
    for od in vc.all_ods:
        for unl in od.unls:
            if unl in organizer_unls:
                i = organizer_unls.index(unl)
                d2 = vc.all_ods[i]
                # if trace: g.trace('found organizer unl:',od.h,'==>',d2.h)
                od.children.append(d2)
                d2.parent_od = od
    # create_organizer_data now ensures od.parent is set.
    for od in vc.all_ods:
        assert od.parent,od.h
    # Extend the descendant lists.
    for od in vc.all_ods:
        vc.compute_descendants(od)
        assert od.descendants is not None
    if trace:
        def tail(head,unl):
            return str(unl[len(head):]) if unl.startswith(head) else str(unl)
        for od in vc.all_ods:
            g.trace(
                '\n  od:',od.h,
                '\n  unl:',od.unl,
                '\n  unls:', [tail(od.unl,z) for z in od.unls],
                '\n  source (unl):',od.source_unl or repr(''),
                '\n  parent (pos):', od.parent.h,
                '\n  children:',[z.h for z in od.children],
                '\n  descendants:',[str(z.h) for z in od.descendants])
#@+node:ekr.20150312225028.64: *8* vc.compute_descendants
def compute_descendants(self,od,level=0,result=None):
    '''Compute the descendant od nodes of od.'''
    trace = False # and not g.unitTesting
    vc = self
    if level == 0:
        result = []
    if od.descendants is None:
        for child in od.children:
            result.append(child)
            result.extend(vc.compute_descendants(child,level+1,result))
            result = list(set(result))
        if level == 0:
            od.descendants = result
            if trace: g.trace(od.h,[z.h for z in result])
        return result
    else:
        if trace: g.trace('cached',od.h,[z.h for z in od.descendants])
        return od.descendants
#@+node:ekr.20150312225028.65: *7* 5: vc.compute_all_organized_positions
def compute_all_organized_positions(self,root):
    '''Compute the list of positions organized by every od.'''
    trace = False and not g.unitTesting
    vc = self
    for od in vc.all_ods:
        if od.unls:
            # Do a full search only for the first unl.
            # parent = vc.find_position_for_relative_unl(root,od.unls[0])
            if True: # parent:
                for unl in od.unls:
                    p = vc.find_position_for_relative_unl(root,unl)
                    # p = vc.find_position_for_relative_unl(parent,vc.unl_tail(unl))
                    if p:
                        od.organized_nodes.append(p.copy())
                    if trace: g.trace('exists:',od.exists,
                        'od:',od.h,'unl:',unl,
                        'p:',p and p.h or '===== None')
            else:
                g.trace('fail',od.unls[0])
#@+node:ekr.20150312225028.66: *7* 6: vc.create_anchors_d
def create_anchors_d (self):
    '''
    Create vc.anchors_d.
    Keys are positions, values are lists of ods having that anchor.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = {}
    if trace: g.trace('all_ods',[z.h for z in vc.all_ods])
    for od in vc.all_ods:
        # Compute the anchor if it does not yet exists.
        # Valid now that p.__hash__ exists.
        key = od.anchor
        # key = '.'.join([str(z) for z in od.anchor.sort_key(od.anchor)])
        # key = '%s (%s)' % (key,od.anchor.h)
        aList = d.get(key,[])
        # g.trace(od.h,od.anchor.h,key,aList)
        aList.append(od)
        d[key] = aList
    if trace:
        for key in sorted(d.keys()):
            g.trace('od.anchor: %s ods: [%s]' % (key.h,','.join(z.h for z in d.get(key))))
    vc.anchors_d = d
#@+node:ekr.20150312225028.67: *6* vc.demote & helpers
def demote(self,root):
    '''
    The main line of the @auto-view algorithm. Traverse root's entire tree,
    placing items on the global work list.
    '''
    trace = False # and not g.unitTesting
    trace_loop = True
    vc = self
    active = None # The active od.
    vc.pending = [] # Lists of pending demotions.
    d = vc.anchor_offset_d # For traces.
    for p in root.subtree():
        parent = p.parent()
        if trace and trace_loop:
            if 1:
                g.trace('-----',p.childIndex(),p.h)
            else:
                g.trace(
                    '=====\np:',p.h,
                    'childIndex',p.childIndex(),
                    '\nparent:',parent.h,
                    'parent:offset',d.get(parent,0))
        vc.n_nodes_scanned += 1
        vc.terminate_organizers(active,parent)
        found = vc.find_organizer(parent,p)
        if found:
            pass # vc.enter_organizers(found,p)
        else:
            pass # vc.terminate_all_open_organizers()
        if trace and trace_loop:
            g.trace(
                'active:',active and active.h or 'None',
                'found:',found and found.h or 'None')
        # The main case statement...
        if found is None and active:
            vc.add_to_pending(active,p)
        elif found is None and not active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(None,p)
        elif found and found == active:
            # Pending nodes *will* be organized.
            for z in vc.pending:
                active2,child2 = z
                vc.add(active2,child2,'found==active:pending')
            vc.pending = []
            vc.add(active,p,'found==active')
        elif found and found != active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(found,p)
            active = found
            vc.enter_organizers(found,p)
            vc.add(active,p,'found!=active')
        else: assert False,'can not happen'
#@+node:ekr.20150312225028.68: *7* vc.add
def add(self,active,p,tag):
    '''
    Add p, an existing (imported) node to the global work list.
    Subtract 1 from the vc.anchor_offset_d entry for p.parent().
    
    Exception: do *nothing* if p is a child of an existing organizer node.
    '''
    trace = False # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(active,g.callers())
    if active.p == p.parent() and active.exists:
        if trace and verbose: g.trace('===== do nothing',active.h,p.h)
    else:
        data = active.p,p.copy()
        vc.add_to_work_list(data,tag)
        vc.anchor_decr(anchor=p.parent(),p=p)
        
#@+node:ekr.20150312225028.69: *7* vc.add_organizer_node
def add_organizer_node (self,od,p):
    '''
    Add od to the appropriate move list.
    p is the existing node that caused od to be added.
    '''
    trace = True # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(od.h,'parent',od.parent_od and od.parent_od.h or 'None')
    if od.parent_od:
        # Not a bare organizer: a child of another organizer node.
        # If this is an existing organizer, it's *position* may have
        # been moved without active.moved being set.
        data = od.parent_od.p,od.p
        if data in vc.work_list:
            if trace and verbose: g.trace(
                '**** duplicate 1: setting moved bit.',od.h)
            od.moved = True
        elif od.parent_od.exists:    
            anchor = od.parent_od.p
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            # g.trace('anchor:',anchor.h,'p:',p.h,'childIndex',p.childIndex())
            vc.add_to_bare_list(data,'non-bare existing')
        else:
            vc.add_to_work_list(data,'non-bare')
    elif od.p == od.anchor:
        if trace and verbose: g.trace(
            '***** existing organizer: do not move:',od.h)
    else:
        # This can be pre-computed?
        bare_list = [p for parent,p,n in vc.global_bare_organizer_node_list]
        if od.p in bare_list:
            if trace and verbose: g.trace(
                '**** duplicate 2: setting moved bit.',od.h)
            od.moved = True
        else:
            # A bare organizer node: a child of an *ordinary* node.
            anchor = p.parent()
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            vc.add_to_bare_list(data,'bare')
#@+node:ekr.20150312225028.70: *7* vc.add_to_bare_list
def add_to_bare_list(self,data,tag):
    '''Add data to the bare organizer list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    # Prevent duplicagtes.
    anchor,p,n = data
    for data2 in vc.global_bare_organizer_node_list:
        a2,p2,n2 = data2
        if p == p2:
            if trace: g.trace('ignore duplicate',
                'n:',n,anchor.h,'==>',p.h)
            return
    vc.global_bare_organizer_node_list.append(data)
    if trace:
        anchor,p,n = data
        g.trace('=====',tag,'n:',n,anchor.h,'==>',p.h)
            # '\n  anchor:',anchor.h,
            # '\n  p:',p.h)
#@+node:ekr.20150312225028.71: *7* vc.add_to_pending
def add_to_pending(self,active,p):
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace(active.p.h,'==>',p.h)
    vc.pending.append((active,p.copy()),)
#@+node:ekr.20150312225028.72: *7* vc.add_to_work_list
def add_to_work_list(self,data,tag):
    '''Append the data to the work list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.work_list.append(data)
    if trace:
        active,p = data
        g.trace('=====',tag,active.h,'==>',p.h)
#@+node:ekr.20150312225028.73: *7* vc.anchor_decr
def anchor_decr(self,anchor,p): # p is only for traces.
    '''
    Decrement the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n - 1
    if trace: g.trace(n-1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.74: *7* vc.anchor_incr
def anchor_incr(self,anchor,p): # p is only for traces.
    '''
    Increment the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n + 1
    if trace: g.trace(n+1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.75: *7* vc.clear_pending
def clear_pending(self,active,p):
    '''Clear the appropriate entries from the pending list.'''
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace('===== clear pending',len(vc.pending))
    if False: # active and active.parent_od:
        for data in vc.pending:
            data = active.parent_od.p,data[1]
            vc.add_to_work_list(data,'clear-pending-to-active')
    vc.pending = []
#@+node:ekr.20150312225028.76: *7* vc.enter_organizers
def enter_organizers(self,od,p):
    '''Enter all organizers whose anchors are p.'''
    vc = self
    ods = []
    while od:
        ods.append(od)
        od = od.parent_od
    if ods:
        for od in reversed(ods):
            vc.add_organizer_node(od,p)
#@+node:ekr.20150312225028.77: *7* vc.find_organizer
def find_organizer(self,parent,p):
    '''Return the organizer that organizers p, if any.'''
    trace = False # and not g.unitTesting
    vc = self
    anchor = parent
    ods = vc.anchors_d.get(anchor,[])
    for od in ods:
        if p in od.organized_nodes:
            if trace: g.trace('found:',od.h,'for',p.h)
            return od
    return None
#@+node:ekr.20150312225028.78: *7* vc.terminate_organizers
def terminate_organizers(self,active,p):
    '''Terminate all organizers whose anchors are not ancestors of p.'''
    trace = False # and not g.unitTesting
    od = active
    while od and od.anchor != p and od.anchor.isAncestorOf(p):
        if not od.closed:
            if trace: g.trace('===== closing',od.h)
            od.closed = True
        od = od.parent_od
#@+node:ekr.20150312225028.79: *7* vc.terminate_all_open_organizers
def terminate_all_open_organizers(self):
    '''Terminate all open organizers.'''
    trace = True # and not g.unitTesting
    if 0:
        g.trace()
        for od in self.all_ods:
            if od.opened and not od.closed:
                if trace: g.trace('===== closing',od.h)
                od.closed = True
#@+node:ekr.20150312225028.80: *6* vc.move_nodes & helpers
def move_nodes(self):
    '''Move nodes to their final location and delete the temp node.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.move_nodes_to_organizers(trace)
    vc.move_bare_organizers(trace)
    vc.temp_node.doDelete()
#@+node:ekr.20150312225028.81: *7* vc.move_nodes_to_organizers
def move_nodes_to_organizers(self,trace):
    '''Move all nodes in the work_list.'''
    trace = False # and not g.unitTesting
    trace_dict = False
    trace_moves = False
    trace_deletes = False
    vc = self
    if trace: # A highly useful trace!
        g.trace('\n\nunsorted_list...\n%s' % (
            '\n'.join(['%40s ==> %s' % (parent.h,p.h)
                for parent,p in vc.work_list])))
    # Create a dictionary of each organizers children.
    d = {}
    for parent,p in vc.work_list:
        # This key must remain stable if parent moves.
        key = parent
        aList = d.get(key,[])
        aList.append(p)
        # g.trace(key,[z.h for z in aList])
        d[key] = aList
    if trace and trace_dict:
        # g.trace('d...',sorted([z.h for z in d.keys()]))
        g.trace('d{}...')
        for key in sorted(d.keys()):
            aList = [z.h for z in d.get(key)]
            g.trace('%s %-20s %s' % (id(key),key.h,vc.dump_list(aList,indent=29)))
    # Move *copies* of non-organizer nodes to each organizer.
    organizers = list(d.keys())
    existing_organizers = [z.p.copy() for z in vc.existing_ods]
    moved_existing_organizers = {} # Keys are vnodes, values are positions.
    for parent in organizers:
        aList = d.get(parent,[])
        if trace and trace_moves:
            g.trace('===== moving/copying:',parent.h,
                'with %s children:' % (len(aList)),
                '\n  '+'\n  '.join([z.h for z in aList]))
        for p in aList:
            if p in existing_organizers:
                if trace and trace_moves:
                    g.trace('copying existing organizer:',p.h)
                    g.trace('children:',
                    '\n  '+'\n  '.join([z.h for z in p.children()]))
                copy = vc.copy_tree_to_last_child_of(p,parent)
                old = moved_existing_organizers.get(p.v)
                if old and trace_moves:
                    g.trace('*********** overwrite',p.h)
                moved_existing_organizers[p.v] = copy
            elif p in organizers:
                if trace and trace_moves:
                    g.trace('moving organizer:',p.h)
                aList = d.get(p)
                if aList:
                    if trace and trace_moves: g.trace('**** relocating',
                        p.h,'children:',
                        '\n  '+'\n  '.join([z.h for z in p.children()]))
                    del d[p]
                p.moveToLastChildOf(parent)
                if aList:
                    d[p] = aList
            else:
                parent2 = moved_existing_organizers.get(parent.v)
                if parent2:
                    if trace and trace_moves:
                        g.trace('***** copying to relocated parent:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent2)
                else:
                    if trace and trace_moves: g.trace('copying:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent)
    # Finally, delete all the non-organizer nodes, in reverse outline order.
    def sort_key(od):
        parent,p = od
        return p.sort_key(p)
    sorted_list = sorted(vc.work_list,key=sort_key)
    if trace and trace_deletes:
        g.trace('===== deleting nodes in reverse outline order...')
    for parent,p in reversed(sorted_list):
        if p.v in moved_existing_organizers:
            if trace and trace_deletes:
                g.trace('deleting moved existing organizer:',p.h)
            p.doDelete()
        elif p not in organizers:
            if trace and trace_deletes:
                g.trace('deleting non-organizer:',p.h)
            p.doDelete()
#@+node:ekr.20150312225028.82: *7* vc.move_bare_organizers
def move_bare_organizers(self,trace):
    '''Move all nodes in global_bare_organizer_node_list.'''
    trace = False # and not g.unitTesting
    trace_data = True
    trace_move = True
    vc = self
    # For each parent, sort nodes on n.
    d = {} # Keys are vnodes, values are lists of tuples (n,parent,p)
    existing_organizers = [od.p for od in vc.existing_ods]
    if trace: g.trace('ignoring existing organizers:',
        [p.h for p in existing_organizers])
    for parent,p,n in vc.global_bare_organizer_node_list:
        if p not in existing_organizers:
            key = parent.v
            aList = d.get(key,[])
            if (parent,p,n) not in aList:
                aList.append((parent,p,n),)
                d[key] = aList
    # For each parent, add nodes in childIndex order.
    def key_func(obj):
        return obj[0]
    for key in d.keys():
        aList = d.get(key)
        for data in sorted(aList,key=key_func):
            parent,p,n = data
            n2 = parent.numberOfChildren()
            if trace and trace_data:
                g.trace(n,parent.h,'==>',p.h)
            if trace and trace_move: g.trace(
                'move: %-20s:' % (p.h),
                'to child: %2s' % (n),
                'of: %-20s' % (parent.h),
                'with:',n2,'children')
            p.moveToNthChildOf(parent,n)
#@+node:ekr.20150312225028.83: *7* vc.copy_tree_to_last_child_of
def copy_tree_to_last_child_of(self,p,parent):
    '''Copy p's tree to the last child of parent.'''
    vc = self
    assert p != parent,p
        # A failed assert leads to unbounded recursion.
    # print('copy_tree_to_last_child_of',p.h,parent.h)
    root = parent.insertAsLastChild()
    root.b,root.h = p.b,p.h
    root.v.u = copy.deepcopy(p.v.u)
    for child in p.children():
        vc.copy_tree_to_last_child_of(child,root)
    return root
#@+node:ekr.20150312225028.84: *5* vc.Helpers
#@+node:ekr.20150312225028.85: *6* vc.at_auto_view_body and match_at_auto_body
def at_auto_view_body(self,p):
    '''Return the body text for the @auto-view node for p.'''
    # Note: the unl of p relative to p is simply p.h,
    # so it is pointless to add that to the @auto-view node.
    return 'gnx: %s\n' % p.v.gnx

def match_at_auto_body(self,p,auto_view):
    '''Return True if any line of auto_view.b matches the expected gnx line.'''
    if 0: g.trace(p.b == 'gnx: %s\n' % auto_view.v.gnx,
        g.shortFileName(p.h),auto_view.v.gnx,p.b.strip())
    return p.b == 'gnx: %s\n' % auto_view.v.gnx
#@+node:ekr.20150312225028.86: *6* vc.clean_nodes (not used)
def clean_nodes(self):
    '''Delete @auto-view nodes with no corresponding @auto nodes.'''
    vc = self
    c = vc.c
    views = vc.has_at_views_node()
    if not views:
        return
    # Remember the gnx of all @auto nodes.
    d = {}
    for p in c.all_unique_positions():
        if vc.is_at_auto_node(p):
            d[p.v.gnx] = True
    # Remember all unused @auto-view nodes.
    delete = []
    for child in views.children():
        s = child.b and g.splitlines(child.b)
        gnx = s[len('gnx'):].strip()
        if gnx not in d:
            g.trace(child.h,gnx)
            delete.append(child.copy())
    for p in reversed(delete):
        p.doDelete()
    c.selectPosition(views)
#@+node:ekr.20150312225028.87: *6* vc.comments...
#@+node:ekr.20150312225028.88: *7* vc.comment_delims
def comment_delims(self,p):
    '''Return the comment delimiter in effect at p, an @auto node.'''
    vc = self
    c = vc.c
    d = g.get_directives_dict(p)
    s = d.get('language') or c.target_language
    language,single,start,end = g.set_language(s,0)
    return single,start,end
#@+node:ekr.20150312225028.89: *7* vc.delete_leading_comments
def delete_leading_comments(self,delims,p):
    '''
    Scan for leading comments from p and return them.
    At present, this only works for single-line comments.
    '''
    single,start,end = delims
    if single:
        lines = g.splitLines(p.b)
        result = []
        for s in lines:
            if s.strip().startswith(single):
                result.append(s)
            else: break
        if result:
            p.b = ''.join(lines[len(result):])
            # g.trace('len(result)',len(result),p.h)
            return ''.join(result)
    return None
#@+node:ekr.20150312225028.90: *7* vc.is_comment_node
def is_comment_node(self,p,root,delims=None):
    '''Return True if p.b contains nothing but comments or blank lines.'''
    vc = self
    if not delims:
        delims = vc.comment_delims(root)
    # pylint: disable=unpacking-non-sequence
    single,start,end = delims
    assert single or start and end,'bad delims: %r %r %r' % (single,start,end)
    if single:
        for s in g.splitLines(p.b):
            s = s.strip()
            if s and not s.startswith(single) and not g.isDirective(s):
                return False
        return True
    else:
        def check_comment(s):
            done,in_comment = False,True
            i = s.find(end)
            if i > -1:
                tail = s[i+len(end):].strip()
                if tail: done = True
                else: in_comment = False
            return done,in_comment
        
        done,in_comment = False,False
        for s in g.splitLines(p.b):
            s = s.strip()
            if not s:
                pass
            elif in_comment:
                done,in_comment = check_comment(s)
            elif g.isDirective(s):
                pass
            elif s.startswith(start):
                done,in_comment = check_comment(s[len(start):])
            else:
                # g.trace('fail 1: %r %r %r...\n%s' % (single,start,end,s)
                return False
            if done:
                return False
        # All lines pass.
        return True
#@+node:ekr.20150312225028.91: *7* vc.is_comment_organizer_node
# def is_comment_organizer_node(self,p,root):
    # '''
    # Return True if p is an organizer node in the given @auto tree.
    # '''
    # return p.hasChildren() and vc.is_comment_node(p,root)
#@+node:ekr.20150312225028.92: *7* vc.post_move_comments
def post_move_comments(self,root):
    '''Move comments from the start of nodes to their parent organizer node.'''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    for p in root.subtree():
        if p.hasChildren() and not p.b:
            s = vc.delete_leading_comments(delims,p.firstChild())
            if s:
                p.b = s
                # g.trace(p.h)
#@+node:ekr.20150312225028.93: *7* vc.pre_move_comments
def pre_move_comments(self,root):
    '''
    Move comments from comment nodes to the next node.
    This must be done before any other processing.
    '''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    aList = []
    for p in root.subtree():
        if p.hasNext() and vc.is_comment_node(p,root,delims=delims):
            aList.append(p.copy())
            next = p.next()
            if p.b: next.b = p.b + next.b
    # g.trace([z.h for z in aList])
    c.deletePositionsInList(aList)
        # This sets c.changed.
#@+node:ekr.20150312225028.94: *6* vc.find...
# The find commands create the node if not found.
#@+node:ekr.20150312225028.95: *7* vc.find_absolute_unl_node
def find_absolute_unl_node(self,unl,priority_header=False):
    '''Return a node matching the given absolute unl.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    vc = self
    aList = unl.split('-->')
    if aList:
        first,rest = aList[0],'-->'.join(aList[1:])
        count = 0
        pos = re.findall(pos_pattern,first)
        nth_sib,pos = pos[0] if pos else (0,0)
        pos = int(pos) if pos else 0
        nth_sib = int(nth_sib)
        first = re.sub(pos_pattern,"",first).replace('--%3E','-->')
        for parent in vc.c.rootPosition().self_and_siblings():
            if parent.h.strip() == first.strip():
                if pos == count:
                    if rest:
                        return vc.find_position_for_relative_unl(parent,rest,priority_header=priority_header)
                    else:
                        return parent
                count = count+1
        #Here we could find and return the nth_sib if an exact header match was not found
    return None
#@+node:ekr.20150312225028.96: *7* vc.find_at_auto_view_node & helper
def find_at_auto_view_node (self,root):
    '''
    Return the @auto-view node for root, an @auto node.
    Create the node if it does not exist.
    '''
    vc = self
    views = vc.find_at_views_node()
    p = vc.has_at_auto_view_node(root)
    if not p:
        p = views.insertAsLastChild()
        p.h = '@auto-view:' + root.h[len('@auto'):].strip()
        p.b = vc.at_auto_view_body(root)
    return p
#@+node:ekr.20150312225028.97: *7* vc.find_clones_node
def find_at_clones_node(self,root):
    '''
    Find the @clones node for root, an @auto node.
    Create the @clones node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@clones'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.98: *7* vc.find_at_headlines_node
def find_at_headlines_node(self,root):
    '''
    Find the @headlines node for root, an @auto node.
    Create the @headlines node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@headlines'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.99: *7* vc.find_gnx_node
def find_gnx_node(self,gnx):
    '''Return the first position having the given gnx.'''
    # This is part of the read logic, so newly-imported
    # nodes will never have the given gnx.
    vc = self
    for p in vc.c.all_unique_positions():
        if p.v.gnx == gnx:
            return p
    return None
#@+node:ekr.20150312225028.100: *7* vc.find_organizers_node
def find_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, and @auto node.
    Create the @organizers node if it doesn't exist.
    '''
    vc = self
    c = vc.c
    h = '@organizers'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.101: *7* vc.find_position_for_relative_unl
def find_position_for_relative_unl(self,parent,unl,priority_header=False):
    '''
    Return the node in parent's subtree matching the given unl.
    The unl is relative to the parent position.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    # This is called from finish_create_organizers & compute_all_organized_positions.
    trace = False # and not g.unitTesting
    trace_loop = True
    trace_success = False
    vc = self
    if not unl:
        if trace and trace_success:
            g.trace('return parent for empty unl:',parent.h)
        return parent
    # The new, simpler way: drop components of the unl automatically.
    drop,p = [],parent # for debugging.
    # if trace: g.trace('p:',p.h,'unl:',unl)
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    for s in unl.split('-->'):
        found = False # The last part must match.
        if 1:
            # Create the list of children on the fly.
            aList = vc.headlines_dict.get(p.v)
            if aList is None:
                aList = [z.h for z in p.children()]
                vc.headlines_dict[p.v] = aList
            try:
                pos = re.findall(pos_pattern,s)
                nth_sib,pos = pos[0] if pos else (0,0)
                pos = int(pos) if pos else 0
                nth_sib = int(nth_sib)
                s = re.sub(pos_pattern,"",s).replace('--%3E','-->')
                indices = [i for i, x in enumerate(aList) if x == s]
                if len(indices)>pos:
                    #First we try the nth node with same header
                    n = indices[pos]
                    p = p.nthChild(n)
                    found = True
                elif len(indices)>0:
                    #Then we try any node with same header
                    n = indices[-1]
                    p = p.nthChild(n)
                    found = True
                elif not priority_header:
                    #Then we go for the child index if return_pos is true
                    if len(aList)>nth_sib:
                        n = nth_sib
                    else:
                        n = len(aList)-1
                    if n>-1:
                        p = p.nthChild(n)
                    else:
                        g.es('Partial UNL match: Referenced level is higher than '+str(p.level()))
                    found = True
                if trace and trace_loop: g.trace('match:',s)
            except ValueError: # s not in aList.
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
        else: # old code.
            for child in p.children():
                if child.h == s:
                    p = child
                    found = True
                    if trace and trace_loop: g.trace('match:',s)
                    break
                # elif trace and trace_loop: g.trace('no match:',child.h)
            else:
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
    if not found and priority_header:
        aList = []
        for p in vc.c.all_unique_positions():
            if p.h.replace('--%3E','-->') in unl:
                aList.append((p.copy(),p.get_UNL(False,False,True)))
        unl_list = [re.sub(pos_pattern,"",x).replace('--%3E','-->') for x in unl.split('-->')]
        for iter_unl in aList:
            maxcount = 0
            count = 0
            compare_list = unl_list[:]
            for header in reversed(iter_unl[1].split('-->')):
                if re.sub(pos_pattern,"",header).replace('--%3E','-->') == compare_list[-1]:
                    count = count+1
                    compare_list.pop(-1)
                else:
                    break
            if count > maxcount:
                p = iter_unl[0]
                found = True
    if found:
        if trace and trace_success:
            g.trace('found unl:',unl,'parent:',p.h,'drop',drop)
    else:
        if trace: g.trace('===== unl not found:',unl,'parent:',p.h,'drop',drop)
    return p if found else None
#@+node:ekr.20150312225028.102: *7* vc.find_representative_node
def find_representative_node (self,root,target):
    '''
    root is an @auto node. target is a clones node within root's tree.
    Return a node *outside* of root's tree that is cloned to target,
    preferring nodes outside any @<file> tree.
    Never return any node in any @views or @view tree.
    '''
    trace = False and not g.unitTesting
    assert target
    assert root
    vc = self
    # Pass 1: accept only nodes outside any @file tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p.isAnyAtFileNode():
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 1:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    # Pass 2: accept any node outside the root tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p == root:
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 2:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    g.trace('no representative node for:',target,'parent:',target.parent())
    return None
#@+node:ekr.20150312225028.103: *7* vc.find_views_node
def find_at_views_node(self):
    '''
    Find the first @views node in the outline.
    If it does not exist, create it as the *last* top-level node,
    so that no existing positions become invalid.
    '''
    vc = self
    c = vc.c
    p = g.findNodeAnywhere(c,'@views')
    if not p:
        last = c.rootPosition()
        while last.hasNext():
            last.moveToNext()
        p = last.insertAfter()
        p.h = '@views'
        # c.selectPosition(p)
        # c.redraw()
    return p
#@+node:ekr.20150312225028.104: *6* vc.has...
# The has commands return None if the node does not exist.
#@+node:ekr.20150312225028.105: *7* vc.has_at_auto_view_node
def has_at_auto_view_node(self,root):
    '''
    Return the @auto-view node corresponding to root, an @root node.
    Return None if no such node exists.
    '''
    vc = self
    c = vc.c
    assert vc.is_at_auto_node(root) or vc.is_at_file_node(root),root
    views = g.findNodeAnywhere(c,'@views')
    if views:
        # Find a direct child of views with matching headline and body.
        for p in views.children():
            if vc.match_at_auto_body(p,root):
                return p
    return None
#@+node:ekr.20150312225028.106: *7* vc.has_clones_node
def has_at_clones_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@clones')
#@+node:ekr.20150312225028.107: *7* vc.has_at_headlines_node
def has_at_headlines_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@headlines')
#@+node:ekr.20150312225028.108: *7* vc.has_organizers_node
def has_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, an @auto node.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@organizers')
#@+node:ekr.20150312225028.109: *7* vc.has_views_node
def has_at_views_node(self):
    '''Return the @views or None if it does not exist.'''
    vc = self
    return g.findNodeAnywhere(vc.c,'@views')
#@+node:ekr.20150312225028.110: *6* vc.is...
#@+node:ekr.20150312225028.111: *7* vc.is_at_auto_node
def is_at_auto_node(self,p):
    '''Return True if p is an @auto node.'''
    return g.match_word(p.h,0,'@auto') and not g.match(p.h,0,'@auto-')
        # Does not match @auto-rst, etc.

def is_at_file_node(self,p):
    '''Return True if p is an @file node.'''
    return g.match_word(p.h,0,'@file')
#@+node:ekr.20150312225028.112: *7* vc.is_cloned_outside_parent_tree
def is_cloned_outside_parent_tree(self,p):
    '''Return True if a clone of p exists outside the tree of p.parent().'''
    return len(list(set(p.v.parents))) > 1
#@+node:ekr.20150312225028.113: *7* vc.is_organizer_node
def is_organizer_node(self,p,root):
    '''
    Return True if p is an organizer node in the given @auto tree.
    '''
    vc = self
    return p.hasChildren() and vc.is_comment_node(p,root)

#@+node:ekr.20150312225028.114: *6* vc.testing...
#@+node:ekr.20150312225028.115: *7* vc.compare_test_trees
def compare_test_trees(self,root1,root2):
    '''
    Compare the subtrees whose roots are given.
    This is called only from unit tests.
    '''
    vc = self
    s1,s2 = vc.trial_write(root1),vc.trial_write(root2)
    if s1 == s2:
        return True
    g.trace('Compare:',root1.h,root2.h)
    p2 = root2.copy().moveToThreadNext()
    for p1 in root1.subtree():
        if p1.h == p2.h:
            g.trace('Match:',p1.h)
        else:
            g.trace('Fail: %s != %s' % (p1.h,p2.h))
            break
        p2.moveToThreadNext()
    return False
#@+node:ekr.20150312225028.116: *7* vc.compare_trial_writes
def compare_trial_writes(self,s1,s2):
    '''
    Compare the two strings, the results of trial writes.
    Stop the comparison after the first mismatch.
    '''
    trace_matches = False
    full_compare = False
    lines1,lines2 = g.splitLines(s1),g.splitLines(s2)
    i,n1,n2 = 0,len(lines1),len(lines2)
    while i < n1 and i < n2:
        s1,s2 = lines1[i].rstrip(),lines2[i].rstrip()
        i += 1
        if s1 == s2:
            if trace_matches: g.trace('Match:',s1)
        else:
            g.trace('Fail:  %s != %s' % (s1,s2))
            if not full_compare: return
    if i < n1:
        g.trace('Extra line 1:',lines1[i])
    if i < n2:
        g.trace('Extra line 2:',lines2[i])
#@+node:ekr.20150312225028.117: *7* vc.dump_list
def dump_list(self,aList,indent=4):
    '''Dump a list, one item per line.'''
    lead = '\n' + ' '*indent
    return lead+lead.join(sorted(aList))
#@+node:ekr.20150312225028.118: *7* vc.trial_write
def trial_write(self,root):
    '''
    Return a trial write of outline whose root is given.
    
    **Important**: the @auto import and write code end all nodes with
    newlines. Because no imported nodes are empty, the code below is
    *exactly* equivalent to the @auto write code as far as trailing
    newlines are concerned. Furthermore, we can treat Leo directives as
    ordinary text here.
    '''
    vc = self
    if 1:
        # Do a full trial write, exactly as will be done later.
        at = vc.c.atFileCommands
        ok = at.writeOneAtAutoNode(root,
            toString=True,force=True,trialWrite=True)
        if ok:
            return at.stringOutput
        else:
            g.trace('===== can not happen')
            return ''
    elif 1:
        # Concatenate all body text.  Close, but not exact.
        return ''.join([p.b for p in root.self_and_subtree()])
    else:
        # Compare headlines, ignoring nodes without body text and comment nodes.
        # This was handy during early development.
        return '\n'.join([p.h for p in root.self_and_subtree()
            if p.b and not p.h.startswith('#')])
#@+node:ekr.20150312225028.119: *6* vc.unls...
#@+node:ekr.20150312225028.120: *7* vc.drop_all_organizers_in_unl
def drop_all_organizers_in_unl(self,organizer_unls,unl):
    '''Drop all organizer unl's in unl, recreating the imported unl.'''
    vc = self
    def unl_sort_key(s):
        return s.count('-->')
    for s in reversed(sorted(organizer_unls,key=unl_sort_key)):
        if unl.startswith(s):
            s2 = vc.drop_unl_tail(s)
            unl = s2 + unl[len(s):]
    return unl[3:] if unl.startswith('-->') else unl
#@+node:ekr.20150312225028.121: *7* vc.drop_unl_tail & vc.drop_unl_parent
def drop_unl_tail(self,unl):
    '''Drop the last part of the unl.'''
    return '-->'.join(unl.split('-->')[:-1])

def drop_unl_parent(self,unl):
    '''Drop the penultimate part of the unl.'''
    aList = unl.split('-->')
    return '-->'.join(aList[:-2] + aList[-1:])
#@+node:ekr.20150312225028.122: *7* vc.get_at_organizer_unls
def get_at_organizer_unls(self,p):
    '''Return the unl: lines in an @organizer: node.'''
    return [s[len('unl:'):].strip()
        for s in g.splitLines(p.b)
            if s.startswith('unl:')]

#@+node:ekr.20150312225028.123: *7* vc.relative_unl & unl
def relative_unl(self,p,root):
    '''Return the unl of p relative to the root position.'''
    vc = self
    result = []
    ivar = vc.headline_ivar
    for p in p.self_and_parents():
        if p == root:
            break
        else:
            h = getattr(p.v,ivar,p.h)
            result.append(h)
    return '-->'.join(reversed(result))

def unl(self,p):
    '''Return the unl corresponding to the given position.'''
    vc = self
    return '-->'.join(reversed([
        getattr(p.v,vc.headline_ivar,p.h)
            for p in p.self_and_parents()]))
    # return '-->'.join(reversed([p.h for p in p.self_and_parents()]))
#@+node:ekr.20150312225028.124: *7* vc.source_unl
def source_unl(self,organizer_unls,organizer_unl):
    '''Return the unl of the source node for the given organizer_unl.'''
    vc = self
    return vc.drop_all_organizers_in_unl(organizer_unls,organizer_unl)
#@+node:ekr.20150312225028.125: *7* vc.unl_tail
def unl_tail(self,unl):
    '''Return the last part of a unl.'''
    return unl.split('-->')[:-1][0]
#@+node:ekr.20150312225028.126: *4* vc.Commands
@g.command('view-pack')
def view_pack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.pack()

@g.command('view-unpack')
def view_unpack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.unpack()
        
@g.command('at-file-to-at-auto')
def at_file_to_at_auto_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.convert_at_file_to_at_auto(c.p)
#@+node:ekr.20140711111623.17795: *4* class ConvertController (leoPersistence.py)
class ConvertController(object):
    '''A class to convert @file trees to @auto trees.'''

    def __init__(self, c, p):
        self.c = c
        self.pd = c.persistenceController
        self.root = p.copy()
    @others
#@+node:ekr.20140711111623.17796: *5* convert.delete_at_data_nodes
def delete_at_data_nodes(self, root):
    '''Delete all @data nodes pertaining to root.'''
    cc = self
    pd = cc.pd
    while True:
        p = pd.has_at_data_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20140711111623.17797: *5* convert.import_from_string
def import_from_string(self, s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    # ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    scanner = g.app.scanner_for_ext(c, ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True, c=c, parent=p, s=s)
    p.h = root.h.replace('@file', '@auto' if ok else '@@auto')
    return ok, p
#@+node:ekr.20140711111623.17798: *5* convert.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root, pd = cc.root, c.persistenceController
    # set the expected imported headline for all vnodes.
    t1 = time.time()
    cc.set_expected_imported_headlines(root)
    t2 = time.time()
    # Delete all previous @data nodes for this tree.
    cc.delete_at_data_nodes(root)
    t3 = time.time()
    # Ensure that all nodes of the tree are regularized.
    ok = pd.prepass(root)
    t4 = time.time()
    if not ok:
        g.es_print('Can not convert', root.h, color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
            # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4 - t3),
            '\n  total:                           %4.2f sec' % (t4 - t1))
        return
    # Create the appropriate @data node.
    at_auto_view = pd.update_before_write_foreign_file(root)
    t5 = time.time()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.time()
    if trace and trace_s:
        g.trace('source file...\n', s)
    # Import the @auto string.
    ok, p = cc.import_from_string(s)
    t7 = time.time()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = pd.at_data_body(p)
        # Recreate the organizer nodes, headlines, etc.
        pd.update_after_read_foreign_file(p)
        t8 = time.time()
        # if not ok:
            # p.h = '@@' + p.h
            # g.trace('restoring original @auto file')
            # ok,p = cc.import_from_string(s)
            # if ok:
                # p.h = '@@' + p.h + ' (restored)'
                # if p.next():
                    # p.moveAfter(p.next())
        t9 = time.time()
    else:
        t8 = t9 = time.time()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
        # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4 - t3),
        '\n  update_before_write_foreign_file:%4.2f sec' % (t5 - t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6 - t5),
        '\n  import_from_string:              %4.2f sec' % (t7 - t6),
        '\n  update_after_read_foreign_file   %4.2f sec' % (t8 - t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9 - t8),
        '\n  total:                           %4.2f sec' % (t9 - t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20140711111623.17799: *5* convert.set_expected_imported_headlines
def set_expected_imported_headlines(self, root):
    '''Set v._imported_headline for every vnode.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    aClass = g.app.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic, atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c, root, importing=True)
    fn = c.os_path_finalize_join(theDir, root.h)
    fn = root.h.replace('\\', '/')
    junk, fn = g.os_path_split(fn)
    fn, junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner, 'headlineForNode'):
        for p in root.subtree():
            if not hasattr(p.v, '_imported_headline'):
                h = scanner.headlineForNode(fn, p)
                setattr(p.v, '_imported_headline', h)
                if trace and h != p.h:
                    g.trace('==>', h) # p.h,'==>',h
#@+node:ekr.20140711111623.17800: *5* convert.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind='@file',
        nosentinels=True,
        perfectImportFlag=False,
        scriptWrite=False,
        thinFile=True,
        toString=True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:', ok, 's:...\n' + s)
    return s
#@+node:ekr.20140711111623.17794: *4* pd.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self, root):
    if root.isAtFileNode():
        ConvertController(self.c, root).run()
    else:
        g.es_print('not an @file node:', root.h)
#@+node:ekr.20140131101641.15495: *4* pd.prepass & helper
def prepass(self, root):
    '''Make sure root's tree has no hard-to-handle nodes.'''
    c, pd = self.c, self
    ic = c.importCommands
    ic.tab_width = c.getTabWidth(root)
    language = g.scanForAtLanguage(c, root)
    ext = g.app.language_extension_dict.get(language)
    if not ext: return
    if not ext.startswith('.'): ext = '.' + ext
    scanner = g.app.scanner_for_ext(c, ext)
    if not scanner:
        g.trace('no scanner for', root.h)
        return True # Pretend all went well.
    # Pass 1: determine the nodes to be inserted.
    ok = True
    # parts_list = []
    for p in root.subtree():
        ok2 = pd.regularize_node(p, scanner)
        ok = ok and ok2
    return ok
#@+node:ekr.20140131101641.15496: *5* pd.regularize_node
def regularize_node(self, p, scanner):
    '''Regularize node p so that it will not cause problems.'''
    c = self.c
    ok = scanner(atAuto=True, c=c, parent=p, s=p.b)
        # The scanner is a callback returned by g.app.scanner_for_ext.
        # It must have a c argument.
    if not ok:
        g.es_print('please regularize:', p.h)
    return ok
#@+node:ekr.20150312225028.6: *3* class LogManager (not used yet)
class LogManager:

    '''A class to handle the global log, and especially
    switching the log from commander to commander.'''

    def __init__ (self):

        trace = (False or g.trace_startup) and not g.unitTesting
        if trace: g.es_debug('(LogManager)')

        self.log = None             # The LeoFrame containing the present log.
        self.logInited = False      # False: all log message go to logWaiting list.
        self.logIsLocked = False    # True: no changes to log are allowed.
        self.logWaiting = []        # List of messages waiting to go to a log.
        self.printWaiting = []      # Queue of messages to be sent to the printer.
        self.signon_printed = False # True: the global signon has been printed.

    @others
#@+node:ekr.20150312225028.7: *4* LogM.setLog, lockLog, unlocklog
def setLog (self,log):

    """set the frame to which log messages will go"""

    # print("app.setLog:",log,g.callers())
    if not self.logIsLocked:
        self.log = log

def lockLog(self):
    """Disable changes to the log"""
    self.logIsLocked = True

def unlockLog(self):
    """Enable changes to the log"""
    self.logIsLocked = False
#@+node:ekr.20150312225028.8: *4* LogM.writeWaitingLog
def writeWaitingLog (self,c):
    '''Write all waiting lines to the log.'''
    trace = True
    lm = self
    if trace:
        # Do not call g.es, g.es_print, g.pr or g.trace here!
        print('** writeWaitingLog','silent',g.app.silentMode,c.shortFileName())
        # print('writeWaitingLog',g.callers())
        # import sys ; print('writeWaitingLog: argv',sys.argv)
    if not c or not c.exists:
        return
    if g.unitTesting:
        lm.printWaiting = []
        lm.logWaiting = []
        g.app.setLog(None) # Prepare to requeue for other commanders.
        return
    table = [
        ('Leo Log Window','red'),
        (g.app.signon,'black'),
        (g.app.signon2,'black'),
    ]
    table.reverse()
    c.setLog()
    lm.logInited = True # Prevent recursive call.
    if not lm.signon_printed:
        lm.signon_printed = True
        if not g.app.silentMode:
            print('')
            print('** isPython3: %s' % g.isPython3)
            if not g.enableDB:
                print('** caching disabled')
            print(g.app.signon)
            print(g.app.signon2)
    if not g.app.silentMode:
        for s in lm.printWaiting:
            print(s)
    lm.printWaiting = []
    if not g.app.silentMode:
        for s,color in table:
            lm.logWaiting.insert(0,(s+'\n',color),)
        for s,color in lm.logWaiting:
            g.es('',s,color=color,newline=0)
                # The caller must write the newlines.
    lm.logWaiting = []
    # Essential when opening multiple files...
    lm.setLog(None)
#@+node:ekr.20200915114300.1: *3* Deleted qt docks code
Rev 96f282672de5c11d00 in devel contains the old code.
#@+node:ekr.20200916074952.1: *4* deleted code snippets
#@+node:ekr.20200915153200.1: *5* from << LeoApp: global data >>
    self.defaultWindowState = b'\x00\x00\x00\xff\x00\x00\x00\x00\xfd\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x01\x19\x00\x00\x01^\xfc\x02\x00\x00\x00\x01\xfb\x00\x00\x00\x12\x00d\x00o\x00c\x00k\x00.\x00t\x00a\x00b\x00s\x01\x00\x00\x006\x00\x00\x01^\x00\x00\x00\x1b\x00\xff\xff\xff\x00\x00\x00\x03\x00\x00\x03\x1e\x00\x00\x00\xc3\xfc\x01\x00\x00\x00\x02\xfb\x00\x00\x00\x12\x00d\x00o\x00c\x00k\x00.\x00b\x00o\x00d\x00y\x01\x00\x00\x00\x00\x00\x00\x01\xfd\x00\x00\x001\x00\xff\xff\xff\xfb\x00\x00\x00\x16\x00d\x00o\x00c\x00k\x00.\x00R\x00e\x00n\x00d\x00e\x00r\x01\x00\x00\x02\x05\x00\x00\x01\x19\x00\x00\x001\x00\xff\xff\xff\x00\x00\x01\xfd\x00\x00\x01^\x00\x00\x00\x04\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x08\xfc\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x10\x00i\x00c\x00o\x00n\x00-\x00b\x00a\x00r\x01\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x01\x00\x00\x00$\x00m\x00i\x00n\x00i\x00b\x00u\x00f\x00f\x00e\x00r\x00-\x00t\x00o\x00o\x00l\x00b\x00a\x00r\x01\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
        #
        # For self.restoreWindowState the first time Leo is run.
        # Use the print-window-state to print this value after arranging the docks to your liking.
        # Important: the outline-pane *must* be the central widget.
#@+node:ekr.20200915153256.1: *5* from dw.addNewEditor
    #
    # Create dock, splitting the body dock.
    dock = self.addEditorDock()
    #
    # Create the editor
    widget = qt_text.LeoQTextBrowser(None, c, self)
    widget.setObjectName('richTextEdit')
    wrapper = qt_text.QTextEditWrapper(widget, name='body', c=c)
    self.packLabel(widget)
    #
    # Inject ivars, set bindings, etc.
    inner_frame = self.leo_body_inner_frame
        # Inject ivars *here*, regardless of docking.
    body.injectIvars(inner_frame, name, p, wrapper)
    body.updateInjectedIvars(widget, p)
    wrapper.setAllText(p.b)
    wrapper.see(0)
    c.k.completeAllBindingsForWidget(wrapper)
    if isinstance(widget, QtWidgets.QTextEdit):
        colorizer = leoColorizer.make_colorizer(c, widget, wrapper)
        colorizer.highlighter.setDocument(widget.document())
    else:
        # Scintilla only.
        body.recolorWidget(p, wrapper)
    return dock, wrapper
#@+node:ekr.20200915153424.1: *5* from dw.createBodyFrame
    #
    # Create widgets.
    #
    # bodyFrame has a VGridLayout.
    bodyFrame = self.createFrame(parent, 'bodyFrame')
    grid = self.createGrid(bodyFrame, 'bodyGrid')
    #
    # innerFrame has a VBoxLayout.
    innerFrame = self.createFrame(bodyFrame, 'innerBodyFrame')
    box = self.createVLayout(innerFrame, 'bodyVLayout', spacing=0)
    #
    # Pack the body alone or *within* a LeoLineTextWidget.
    body = self.createText(None, 'richTextEdit')  # A LeoQTextBrowser
    if self.use_gutter:
        lineWidget = qt_text.LeoLineTextWidget(c, body)
        box.addWidget(lineWidget)
    else:
        box.addWidget(body)
    grid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Official ivars
    self.richTextEdit = body
    self.leo_body_frame = bodyFrame
    self.leo_body_inner_frame = innerFrame
    return bodyFrame
#@+node:ekr.20200915153609.1: *5* from dw.setMainWindowOptions
    dw.setDockNestingEnabled(False)
    dw.setDockOptions(
        QtWidgets.QMainWindow.AllowTabbedDocks |
        QtWidgets.QMainWindow.AnimatedDocks)
#@+node:ekr.20200915153647.1: *5* from LeoQtBody.delete_editor_command
    #
    # Sanity checks.
    if not dock:
        dock = w.parent().parent()
        if (getattr(w, 'leo_name', None) == '1' or
            not isinstance(dock, QtWidgets.QDockWidget)
        ):
            g.warning('can not delete main editor')
            return
    #
    # Actually delete the dock.
    if dock in dw.added_editor_docks:
        dw.added_editor_docks.remove(dock)
    dw.removeDockWidget(dock)
        # A QMainWidget method.
    #
    # Select another editor.
    w.leo_label = None
    new_wrapper = list(d.values())[0]
    self.numberOfEditors -= 1
    self.selectEditor(new_wrapper)
#@+node:ekr.20200915153758.1: *5* from LeoQtBody.select/unselectLabel
        if isinstance(label, QtWidgets.QDockWidget):
            # #1517.
            label.setWindowTitle(c.p.h)
        el
#@+node:ekr.20200915153720.1: *5* from LeoQtLog.createTab
        if g.app.dock and c.config.getBool('dockable-log-tabs', default=False):
            # #1154: Support docks in the Log pane.
            dw = c.frame.top
            dock = g.app.gui.create_dock_widget(
                closeable=True, moveable=True, height=50, name=tabName)  # was 100: #1339.
                    # #1207: all plugins docks should be closeable.
            dock.setWidget(contents)
            area = QtCore.Qt.RightDockWidgetArea
            dw.addDockWidget(area, dock)
        else:
#@+node:ekr.20200916075026.1: *4* deleted functions
#@+node:ekr.20200303094809.1: *5*  function:dock_widget
def dock_widget(w):
    """Return the dock widget containing widget w."""
    while w:
        if isinstance(w, QtWidgets.QDockWidget):
            return w
        w = w.parent()
    return None
#@+node:ekr.20200303104851.1: *5* 'hide-body-dock'
@g.command('hide-body-dock')
def hideBodyDock(event):
    '''Hide the Body dock'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.body.widget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105224.1: *5* 'hide-outline-dock'
@g.command('hide-outline-dock')
def hideOutlineDock(event):
    '''Hide the Outline dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.tree.treeWidget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105106.1: *5* 'hide-tabs-dock'
@g.command('hide-tabs-dock')
def hideTabsDock(event):
    '''Hide the Tabs dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.log.logWidget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105513.1: *5* 'show-body-dock'
@g.command('show-body-dock')
def showBodyDock(event):
    '''Show the Body dock'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.body.widget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20200303105712.1: *5* 'show-outline-dock'
@g.command('show-outline-dock')
def showOutlineDock(event):
    '''Show the Outline dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.tree.treeWidget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20200303105622.1: *5* 'show-tabs-dock'
@g.command('show-tabs-dock')
def showTabsDock(event):
    '''Show the Tabs dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.log.logWidget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20190616092159.1: *5* app.get_central_widget
def get_central_widget(self, c):

    assert self.dock, g.callers()
    s = c.config.getString('central-dock-widget')
    if s:
        s = s.lower()
        if s in ('body', 'outline', 'tabs'):
            return s
    return 'outline'
#@+node:ekr.20200305102656.1: *5* app.restoreEditorDockState
def restoreEditorDockState(self, c):

    trace = any([z in g.app.debug for z in ('dock', 'select')]) and not g.app.unitTesting
    tag = 'app.restoreEditorDockState'
    body = c.frame.body
    dw = c.frame.top
    if not dw:
        return
    aps_s= c.db.get('added_editor_aps', '')
    dock_names_s = c.db.get('added_editor_docks', '')
    if not aps_s or not dock_names_s:
        return
    aps = aps_s.split(';')
    dock_names = dock_names_s.split(';')
    if len(aps) != len(dock_names):
        g.trace('can not happen')
        return
    if trace: g.trace('START')
    #
    # #1527: Part 1: Inject leo_wrapper ivar.
    #                Similar to LeoQtBody.injectIvars.
    wrapper = body.wrapper
    w = wrapper.widget
    w.leo_wrapper = wrapper
    if trace:
        print(f"{tag:>30}: {wrapper} <dock for main body>")
    #
    # #1527: Part 2: Pack the Body's label so it tracks p.h.
    #                Similar to code for the 'add-editor' command.
    if body.use_gutter:
        dw.packLabel(w.parent(), n=1)
        w.leo_label = w.parent().leo_label
    else:
        dw.packLabel(w, n=1)
    #
    # Restore all *added* editors.
    d = body.editorWrappers
    for i, dock_name in enumerate(dock_names):
        ap = aps[i]
        f, wrapper = dw.addNewEditor(dock_name)
        d[dock_name] = wrapper
        p = c.archivedPositionToPosition(ap)
        body.injectIvars(
            parentFrame = f,
            name = dock_name,
            p = p,
            wrapper = wrapper,
        )
        w =  c.frame.body.wrapper.widget
        body.updateInjectedIvars(w, p)
        body.selectLabel(wrapper)
        body.selectEditor(wrapper)
        if trace:
            print(f"{tag:>30}: {wrapper} {dock_name}")
    if trace: g.trace('END')
#@+node:ekr.20190528045549.1: *5* app.restoreWindowState
ekr_val = None

def restoreWindowState(self, c):
    """
    Restore the layout of dock widgets and toolbars, using the per-file
    state of the *first* loaded .leo file, or the global state.
    
    Note: The window's position or size has already been restored.
    """
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    tag = 'app.restoreWindowState:'
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    dw = c.frame.top
    if not dw or not hasattr(dw, 'restoreState'):
        if trace: g.trace('no dw.restoreState. dw:', repr(dw))
        return
    # First, restore the editor state.
    self.restoreEditorDockState(c)
    #
    # Support --init-docks.
    # #1196. Let Qt use it's own notion of a default layout.
    #        This should work regardless of the central widget.
    if g.app.init_docks:
        if trace: g.trace('using qt default layout')
        return
    sfn = c.shortFileName()
    table = (
        # First, try the per-outline state.
        (f"windowState:{c.fileName()}", dw.restoreState),
        # Restore the actual window state.
        ('windowState:', dw.restoreState),
    )
    for key, method in table:
        val = self.db.get(key)
        if val:
            if trace:
                g.trace(f"{sfn} found key: {key}")
            try:
                val = base64.decodebytes(val.encode('ascii'))
                    # Elegant pyzo code.
                method(val)
                return
            except Exception as err:
                g.trace(f"{sfn} bad value: {key} {err}")
        # This is not an error.
        elif trace:
            g.trace(f"{sfn} missing key: {key}")
    #
    # #1190 (bad initial layout)
    # Use a pre-defined layout (magic number).
    # The print-window-state prints this magic number for a *given* layout.
    # But this number will work *only* if the central widgets match.
    try:
        central_widget = self.get_central_widget(c)
        # central_widget = c.config.getString('central-dock-widget')
        # if central_widget:
            # central_widget = central_widget.lower()
        if central_widget in (None, 'outline'):
            if trace:
                print(tag, 'using app.defaultWindowState')
            dw.restoreState(self.defaultWindowState)
        elif trace:
            print(tag)
            print('central widget does not match default')
            print('using qt default window state')
        # See print-window-state.
    except Exception:
        g.es_print(tag, 'unexpected exception setting window state')
        g.es_exception()
#@+node:ekr.20190528045643.1: *5* app.saveWindowState
def saveWindowState(self, c):
    """
    Save the window geometry and layout of dock widgets and toolbars.
    
    This is called for all closed windows.
    """
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    dw = c.frame.top
    if not dw or not hasattr(dw, 'saveState'):
        if trace: g.trace('no dw.saveState. dw:', repr(dw))
        return
    table = (
        # Save a default *global* state, for *all* outline files.
        ('windowState:', dw.saveState),
        # Save a per-file state.
        (f"windowState:{c.fileName()}", dw.saveState),
        # Do not save/restore window geometry. That is done elsewhere.
            # (f"windowGeometry:{c.fileName()}" , dw.saveGeometry),
    )
    for key, method in table:
        # This is pyzo code...
        val = method()
            # Method is a QMainWindow method.
        try:
            val = bytes(val)  # PyQt4
        except Exception:
            val = bytes().join(val)  # PySide
        val = base64.encodebytes(val).decode('ascii')
        if trace:
            g.trace(f"{c.shortFileName()} set key: {key}")         
        g.app.db[key] = val
#@+node:ekr.20190613062749.1: *5* debug.print-window-state
@cmd('print-window-state')
def printWindowState(self, event=None):
    """
    For Leo's core developers: print QMainWindow.saveState().
    
    This is the value to be assigned to g.app.defaultWindowState.
    
    Warning: this window state should *only* be used for new users! #1190.
    
    Recommended procedure:

    - Set @bool user-vr-dock = False.  Close Leo.
    - Clear .leo/db caches. Reopen Leo.
    - Make sure Render dock is visible to left of Body dock.
    - Set @bool user-vr-dock = True. Close Leo & reopen.
      The vr dock will be moveable.  Don't move it!
    - Do print-window-state.
    - Change g.app.defaultWindowState.
    """
    c = event.get('c')
    if c:
        print(c.frame.top.saveState())
        if c.config.getBool('dockable-log-tabs', default=False):
            print('Warning: @bool dockable-log-tabs is True')
        central_widget = g.app.get_central_widget(c)
        if central_widget != 'outline':
            print(f"Warning: @string central-dock-widget is {central_widget!r}")
    else:
        print('no c')
#@+node:ekr.20190523115826.1: *5* dw.addEditorDock
added_bodies = 0

def addEditorDock(self, closeable=True, moveable=True):
    """Add an editor dock"""
    #
    # Create the new dock.
    dw, c = self, self.leo_c
    dw.added_bodies += 1
    dock = g.app.gui.create_dock_widget(
        closeable=closeable,
        moveable=moveable,
        height=50,
        name=c.p.h,
    )
    dw.added_editor_docks.append(dock)
    w = dw.createBodyPane(parent=None)
    dock.setWidget(w)
    dw.splitDockWidget(dw.body_dock, dock, QtCore.Qt.Horizontal)
    #
    # monkey-patch dock.closeEvent

    def patched_closeEvent(event=None):
        c.frame.body.delete_editor_command(event, dock=dock)

    dock.closeEvent = patched_closeEvent
    return dock
#@+node:ekr.20190522165123.1: *5* dw.createAllDockWidgets
def createAllDockWidgets(self):
    """Create all the dock widgets."""
    c, dw = self.leo_c, self
    #
    # Compute constants.
    Qt = QtCore.Qt
    bottom, top = Qt.BottomDockWidgetArea, Qt.TopDockWidgetArea
    lt, rt = Qt.LeftDockWidgetArea, Qt.RightDockWidgetArea
    g.placate_pyflakes(bottom, lt, rt, top)
    #
    # Create all the docks.
    central_widget = g.app.get_central_widget(c)
    dockable = c.config.getBool('dockable-log-tabs', default=False)
    table = [
        (True, 50, lt, 'outline', dw.createOutlineDock),  # was 100: #1339.
        (True, 50, bottom, 'body', dw.createBodyPane),  # was 100: #1339.
        (True, 50, rt, 'tabs', dw.createTabsDock),  # was 20: #1339.
        (dockable, 20, rt, 'find', dw.createFindDockOrTab),
        (dockable, 20, rt, 'spell', dw.createSpellDockOrTab),
    ]
    for make_dock, height, area, name, creator in table:
        w = creator(parent=None)
        if not make_dock:
            setattr(dw, f"{name}_dock", None)
            continue
        dock = g.app.gui.create_dock_widget(
            closeable=name != central_widget,
            moveable=name != central_widget,
            height=0,
            name=name)
        dock.setWidget(w)
        # Remember the dock.
        setattr(dw, f"{name}_dock", dock)
        if name == central_widget:
            dw.setCentralWidget(dock)
                # Important: the central widget should be a dock.
            dock.show()  # #1327.
        else:
            dw.addDockWidget(area, dock)
    #
    # Create minibuffer.
    bottom_toolbar = QtWidgets.QToolBar(dw)
    bottom_toolbar.setObjectName('minibuffer-toolbar')
    bottom_toolbar.setWindowTitle('Minibuffer')
    dw.addToolBar(Qt.BottomToolBarArea, bottom_toolbar)
    w = dw.createMiniBuffer(bottom_toolbar)
    bottom_toolbar.addWidget(w)
    #
    # Create other widgets...
    dw.createMenuBar()
    dw.createStatusBar(dw)
#@+node:ekr.20190527120808.1: *5* dw.createFindDockOrTab
def createFindDockOrTab(self, parent):
    """Create a Find dock or tab in the Log pane."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create widgets.
    findTab = QtWidgets.QWidget()
    findTab.setObjectName('findTab')
    findScrollArea = QtWidgets.QScrollArea()
    findScrollArea.setObjectName('findScrollArea')
    #
    # For LeoFind.finishCreate.
    self.findScrollArea = findScrollArea
    self.findTab = findTab
    #
    # Create a tab in the log Dock, if necessary.
    if not c.config.getBool('dockable-log-tabs', default=False):
        self.tabWidget.addTab(findScrollArea, 'Find')
    return findScrollArea
#@+node:ekr.20190528112002.1: *5* dw.createOutlineDock
def createOutlineDock(self, parent):
    """Create the widgets and ivars for Leo's outline."""
    # Create widgets.
    treeFrame = self.createFrame(parent, 'outlineFrame',
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    innerFrame = self.createFrame(treeFrame, 'outlineInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred)
    treeWidget = self.createTreeWidget(innerFrame, 'treeWidget')
    grid = self.createGrid(treeFrame, 'outlineGrid')
    grid.addWidget(innerFrame, 0, 0, 1, 1)
    innerGrid = self.createGrid(innerFrame, 'outlineInnerGrid')
    innerGrid.addWidget(treeWidget, 0, 0, 1, 1)
    # Official ivars...
    self.treeWidget = treeWidget
    return treeFrame
#@+node:ekr.20190527120829.1: *5* dw.createSpellDockOrTab
def createSpellDockOrTab(self, parent):
    """Create a Spell dock  or tab in the Log pane."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create an outer widget.
    spellTab = QtWidgets.QWidget()
    spellTab.setObjectName('docked.spellTab')
    #
    # Create the contents.
    self.createSpellTab(spellTab)
    #
    # Create the Spell tab in the Log dock, if necessary.
    if not c.config.getBool('dockable-log-tabs', default=False):
        tabWidget = self.tabWidget
        tabWidget.addTab(spellTab, 'Spell')
        tabWidget.setCurrentIndex(1)
    return spellTab
#@+node:ekr.20190527163203.1: *5* dw.createTabbedLogDock
def createTabbedLogDock(self, parent):
    """Create a tabbed (legacy) Log dock."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create the log frame.
    logFrame = self.createFrame(parent, 'logFrame',
        vPolicy=QtWidgets.QSizePolicy.Minimum)
    innerFrame = self.createFrame(logFrame, 'logInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred,
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    tabWidget = self.createTabWidget(innerFrame, 'logTabWidget')
    #
    # Pack.
    innerGrid = self.createGrid(innerFrame, 'logInnerGrid')
    innerGrid.addWidget(tabWidget, 0, 0, 1, 1)
    outerGrid = self.createGrid(logFrame, 'logGrid')
    outerGrid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Create the Find tab, embedded in a QScrollArea.
    findScrollArea = QtWidgets.QScrollArea()
    findScrollArea.setObjectName('findScrollArea')
    # Find tab.
    findTab = QtWidgets.QWidget()
    findTab.setObjectName('findTab')
    # Fix #516:
    use_minibuffer = c.config.getBool('minibuffer-find-mode', default=False)
    use_dialog = c.config.getBool('use-find-dialog', default=False)
    if not use_minibuffer and not use_dialog:
        tabWidget.addTab(findScrollArea, 'Find')
    # Do this later, in LeoFind.finishCreate
    self.findScrollArea = findScrollArea
    self.findTab = findTab
    #
    # Spell tab.
    spellTab = QtWidgets.QWidget()
    spellTab.setObjectName('spellTab')
    tabWidget.addTab(spellTab, 'Spell')
    self.createSpellTab(spellTab)
    tabWidget.setCurrentIndex(1)
    #
    # Official ivars
    self.tabWidget = tabWidget  # Used by LeoQtLog.
    return logFrame
#@+node:ekr.20190527121112.1: *5* dw.createTabsDock
def createTabsDock(self, parent):
    """Create the Tabs dock."""
    assert g.app.dock
    assert not parent, repr(parent)
    #
    # Create the log contents
    logFrame = self.createFrame(None, 'logFrame',
        vPolicy=QtWidgets.QSizePolicy.Minimum)
    innerFrame = self.createFrame(logFrame, 'logInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred,
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    tabWidget = self.createTabWidget(innerFrame, 'logTabWidget')
    #
    # Pack. This *is* required.
    innerGrid = self.createGrid(innerFrame, 'logInnerGrid')
    innerGrid.addWidget(tabWidget, 0, 0, 1, 1)
    outerGrid = self.createGrid(logFrame, 'logGrid')
    outerGrid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Official ivars
    self.tabWidget = tabWidget  # Used by LeoQtLog.
    return logFrame
#@+node:ekr.20190724172314.1: *5* qt: show-hide-body-dock
@g.command('show-hide-body-dock')
def show_hide_body_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.body_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'body':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724172258.1: *5* qt: show-hide-outline-dock
@g.command('show-hide-outline-dock')
def show_hide_outline_dock(event):
    """Show or hide the Outline dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.outline_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'outline':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724172547.1: *5* qt: show-hide-render-dock
@g.command('show-hide-render-dock')
def show_hide_render_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    pc = g.app.pluginsController
    vr = pc.getPluginModule('leo.plugins.viewrendered')
    x = vr and vr.controllers.get(c.hash())
    dock = x and x.leo_dock
    if not dock:
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724170436.1: *5* qt: show-hide-tabs-dock
@g.command('show-hide-tabs-dock')
def show_hide_tabs_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.tabs_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'tabs':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190819091950.1: *5* qt_gui.create_dock_widget
total_docks = 0

def create_dock_widget(self, closeable, moveable, height, name):
    """Make a new dock widget in the main window"""
    dock = QtWidgets.QDockWidget(parent=self.main_window)
        # The parent must be a QMainWindow.
    features = dock.DockWidgetFloatable  # #1643.
    # #1643: Widgets are fixed unless --init-docks is in effect
    if moveable and g.app.init_docks:
        features |= dock.DockWidgetMovable
    if closeable:
        features |= dock.DockWidgetClosable
    dock.setFeatures(features)
    dock.setMinimumHeight(height)
    dock.setObjectName(f"dock-{self.total_docks}")
    self.total_docks += 1
    if name:
        dock.setWindowTitle(name.capitalize())
    else:
        # #1527. Suppress the title.
        w = QtWidgets.QWidget()
        dock.setTitleBarWidget(w)
    # #1327: frameFactory.createFrame now ensures that the main window is visible.
    return dock
#@+node:ekr.20200305075130.1: *5* qt_gui.find_dock
def find_dock(self, w):
    """return the QDockWidget containing w, or None"""
    dock = w
    while dock and not isinstance(dock, QtWidgets.QDockWidget):
        dock = dock.parent()
    return dock
#@+node:ekr.20190822113212.1: *5* qt_gui.make_global_outlines_dock
def make_global_outlines_dock(self):
    """
    Create the top-level Outlines (plural) dock,
    containing the 
    The dock's widget will be set later.
    """
    main_window = self.main_window
    # For now, make it the central widget.
    is_central = True
    dock = self.create_dock_widget(
        closeable=not is_central,
        moveable=not is_central,
        height=50,  # was 100: #1339.
        name='',  # #1527: was 'Leo Outlines'
    )
    if is_central:
        main_window.setCentralWidget(dock)
    else:
        area = QtCore.Qt.BottomDockWidgetArea
        main_window.addDockWidget(area, dock)
    return dock
#@+node:ekr.20190601054955.1: *5* qt_gui.raise_dock
def raise_dock(self, widget):
    """Raise the nearest parent QDockWidget, if any."""
    while widget:
        if isinstance(widget, QtWidgets.QDockWidget):
            widget.raise_()
            return
        if not hasattr(widget, 'parent'):
            return
        widget = widget.parent()
#@+node:ekr.20190813161639.1: *3* pyzo_in_leo.py
@first # -*- coding: utf-8 -*-
"""pyzo_in_leo.py: Experimental plugin that adds all of pyzo's features to Leo."""
#
# Easy imports...
import locale
import os
import sys
import threading
from leo.core import leoGlobals as g
from leo.core.leoQt import QtCore, QtGui, QtWidgets

# pylint: disable=import-error
    # pylint doesn't know about the additions to sys.path.
        
# The pyzo global. Set by init()
pyzo = None

# The singleton PyzoController instance.
pyzo_controller = None

@others
@language python
@tabwidth -4
@nobeautify # Indentation of comments is important.
#@+node:ekr.20190930051422.1: *4* Top-level functions (pyzo_in_leo)
#@+node:ekr.20190813161639.4: *5* init (pyzo_in_leo)
init_warning_given = False

def init(): # pyzo_in_leo.py
    '''Return True if this plugin can be loaded.'''
    global pyzo

    from shutil import which
    
    def oops(message):
        global init_warning_given
        if not init_warning_given:
            init_warning_given = True
            print(f"\npyzo_in_leo not loaded: {message}\n")
            g.es('pyzo_in_leo', message, color='red')
        return False
        
    if g.app.gui.guiName() != "qt":
        return oops('requires Qt gui')
    if not getattr(g.app, 'dock'):
        return oops('requires Qt Docks')
    # #1643: This test will never fail now.
        # if not g.app.use_global_docks:
        #     return oops('requires --global-docks')
    #
    # Fail if can't find pyzo.exe.
    pyzo_exec = which('pyzo')
    if not pyzo_exec:
        return oops('can not find pyzo.exe')
    # Add pyzo/source to sys.path
    pyzo_dir = os.path.dirname(pyzo_exec)
    pyzo_source_dir = os.path.join(pyzo_dir, 'source')
    if pyzo_source_dir not in sys.path:
        sys.path.insert(0, pyzo_source_dir)
    # Fail if still can't import pyzo.
    try:
        import pyzo as local_pyzo
        pyzo = local_pyzo
    except ImportError:
        return oops(f"can not import pyzo from {pyzo_source_dir!r}")
    g.plugin_signon(__name__)
    #
    # This replaces MainWindow.closeEvent.
    g.app.pyzo_close_handler = close_handler
        # LeoApp.finishQuit calls this late in Leo's shutdown logic.
    g.registerHandler('after-create-leo-frame', onCreate) 
    return True
#@+node:ekr.20190928061911.1: *5* onCreate
def onCreate(tag, keys): # pyzo_in_leo.py
    """
    Init another commander, and pyzo itself if this is the first commander.
    """
    global pyzo_controller
    c = keys.get('c')
    if not c:
        return
    if not pyzo_controller:
        pyzo_controller = PyzoController()
        pyzo_controller.pyzo_start()
        main_window = g.app.gui.main_window
        main_window.setWindowTitle(c.frame.title)
    pyzo_controller.init_pyzo_menu(c)
#@+node:ekr.20190816163728.1: *5* close_handler
def close_handler(): # pyzo_in_leo.py
    """
    Shut down pyzo.
    
    Called by Leo's shutdown logic when *all* outlines have been closed.
    
    This code is based on MainWindow.closeEvent.
    Copyright (C) 2013-2019 by Almar Klein.
    """

    print('\ng.app.pyzo_close_event\n')
    
    if 1: # EKR: change
    
        def do_nothing(*args, **kwargs):
            pass

        # We must zero this out. pyzo.saveConfig calls this.
        pyzo.main.saveWindowState = do_nothing
    
    # EKR:change-new imports
    from pyzo.core import commandline

    # Are we restaring?
    # restarting = time.time() - self._closeflag < 1.0

    # EKR:change.
    if 1: # As in the original.
        # Save settings
        pyzo.saveConfig()
        pyzo.command_history.save()

    # Stop command server
    commandline.stop_our_server()

    # Proceed with closing...
    pyzo.editors.closeAll()
    
    # EKR:change.
        # # Force the close.
        # if not result:
            # self._closeflag = False
            # event.ignore()
            # return
        # self._closeflag = True

    # Proceed with closing shells
    if 1:
        # pylint: disable=no-member
        pyzo.localKernelManager.terminateAll()
    
    for shell in pyzo.shells:
        shell._context.close()

    if 1: # As in original.
        # Close tools
        for toolname in pyzo.toolManager.getLoadedTools():
            tool = pyzo.toolManager.getTool(toolname)
            tool.close()

    # Stop all threads (this should really only be daemon threads)
        # import threading
    for thread in threading.enumerate():
        if hasattr(thread, 'stop'):
            try:
                thread.stop(0.1)
            except Exception:
                pass

    # EKR:change. Not needed.
        # Proceed as normal
        # QtWidgets.QMainWindow.closeEvent(self, event)
    # EKR:change. Don't exit Leo!
        # if sys.version_info >= (3,3,0): # and not restarting:
            # if hasattr(os, '_exit'):
                # os._exit(0)
#@+node:ekr.20191012094334.1: *5* patched: setShortcut
def setShortcut(self, action): # pyzo_in_leo.py
    """A do-nothing, monkey-patched, version of KeyMapper.setShortcut."""
    pass
#@+node:ekr.20191012093236.1: *5* patched: _get_interpreters_win
def _get_interpreters_win():  # pyzo_in_leo.py
    """
    Monkey-patch pyzo/util/interpreters._get_interpreters_win.

    This patched code fixes an apparent pyzo bug.
    
    Unlike shutil.which, this function returns all plausible python executables.

    Copyright (C) 2013-2019 by Almar Klein.
    """

    import pyzo.util.interpreters as interps ### EKR
    
    found = []

    # Query from registry
    for v in interps.get_interpreters_in_reg(): ### EKR
        found.append(v.installPath() )

    # Check common locations
    for rootname in ['C:/', '~/',
                     'C:/program files/', 'C:/program files (x86)/', 'C:/ProgramData/',
                     '~/appdata/local/programs/python/',
                     '~/appdata/local/continuum/', '~/appdata/local/anaconda/',
                     ]:
        rootname = os.path.expanduser(rootname)
        if not os.path.isdir(rootname):
            continue
        for dname in os.listdir(rootname):
            if dname.lower().startswith(('python', 'pypy', 'miniconda', 'anaconda')):
                found.append(os.path.join(rootname, dname))

    # Normalize all paths, and remove trailing backslashes
    
    ### found = [os.path.normcase(os.path.abspath(v)).strip('\\') for v in found]
    found = [
        os.path.normcase(os.path.abspath(v)).strip('\\') for v in found
            if v is not None ### EKR: Add guard.
    ]

    # Append "python.exe" and check if that file exists
    found2 = []
    for dname in found:
        for fname in ('python.exe', 'pypy.exe'):
            exename = os.path.join(dname, fname)
            if os.path.isfile(exename):
                found2.append(exename)
                break

    # Returnas set (remove duplicates)
    return set(found2)
#@+node:ekr.20190930051034.1: *4* class PyzoController
class PyzoController:
    
    menus_inited = False
    
    @others
#@+node:ekr.20190929180053.1: *5* pz.init_pyzo_menu
def init_pyzo_menu(self, c):
    """
    Add a Pyzo menu to c's menu bar.
    
    This code is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    dw = c.frame.top
    leo_menu_bar = dw.leo_menubar
        # Create the Pyzo menu in *Leo's* per-commander menu bar.
    menuBar = pyzo.main.menuBar()
        # Use *pyzo's* main menuBar to get data.

    # EKR:change-new imports.
    from pyzo import translate
    from pyzo.core.menu import EditMenu, FileMenu, SettingsMenu
        # Testing.
    from pyzo.core.menu import HelpMenu, RunMenu, ShellMenu, ViewMenu
        # Permanent.

    # EKR:change. Create a top-level Pyzo menu.
    pyzoMenu = leo_menu_bar.addMenu("Pyzo")
    menus = [
        # Testing only...
        FileMenu(menuBar, translate("menu", "File")),
        EditMenu(menuBar, translate("menu", "Edit")),
        SettingsMenu(menuBar, translate("menu", "Settings")),
        # Permanent...
        ViewMenu(menuBar, translate("menu", "View")),
        ShellMenu(menuBar, translate("menu", "Shell")),
        RunMenu(menuBar, translate("menu", "Run")),
        RunMenu(menuBar, translate("menu", "Tools")),
        HelpMenu(menuBar, translate("menu", "Help")),
    ]
    menuBar._menumap = {}
    menuBar._menus = menus
    for menu in menuBar._menus:
        pyzoMenu.addMenu(menu)
            # menuBar.addMenu(menu)
        menuName = menu.__class__.__name__.lower().split('menu')[0]
        menuBar._menumap[menuName] = menu

    # Enable tooltips
    def onHover(action):
        # This ugly bit of code makes sure that the tooltip is refreshed
        # (thus raised above the submenu). This happens only once and after
        # ths submenu has become visible.
        if action.menu():
            if not hasattr(menuBar, '_lastAction'):
                menuBar._lastAction = None
                menuBar._haveRaisedTooltip = False
            if action is menuBar._lastAction:
                if ((not menuBar._haveRaisedTooltip) and
                            action.menu().isVisible()):
                    QtWidgets.QToolTip.hideText()
                    menuBar._haveRaisedTooltip = True
            else:
                menuBar._lastAction = action
                menuBar._haveRaisedTooltip = False
        # Set tooltip
        tt = action.statusTip()
        if hasattr(action, '_shortcutsText'):
            tt = tt + ' ({})'.format(action._shortcutsText) # Add shortcuts text in it
        QtWidgets.QToolTip.showText(QtGui.QCursor.pos(), tt)

    menuBar.hovered.connect(onHover)

    if not self.menus_inited:
        self.menus_inited = True
        pyzo.editors.addContextMenu()
        pyzo.shells.addContextMenu()
#@+node:ekr.20190814050859.1: *5* pz.load_all_pyzo_docks
def load_all_pyzo_docks(self):
    """
    Load all pyzo docks into the singleton QMainWindow.
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    assert pyzo.main == g.app.gui.main_window
    tm = pyzo.toolManager
    table = (
        'PyzoFileBrowser',
        'PyzoHistoryViewer',
        'PyzoInteractiveHelp',
        'PyzoLogger',
        'PyzoSourceStructure',
        'PyzoWebBrowser',
        'PyzoWorkspace',
    )
    for tool_id in table:
        tm.loadTool(tool_id)
        
    # EKR-change: old code.
        # # Load tools
        # if pyzo.config.state.newUser and not pyzo.config.state.loadedTools:
            # pyzo.toolManager.loadTool('pyzosourcestructure')
            # pyzo.toolManager.loadTool('pyzofilebrowser', 'pyzosourcestructure')
        # elif pyzo.config.state.loadedTools:
            # for toolId in pyzo.config.state.loadedTools:
                # pyzo.toolManager.loadTool(toolId)
#@+node:ekr.20190816131753.1: *5* pz.main_window_ctor
def main_window_ctor(self):
    """
    Simulate MainWindow.__init__().
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """

    # print('\nBEGIN main_window_ctor\n')
    
    # EKR:change. New imports
    import pyzo.core.main as main
    from pyzo.core import commandline
    
    # EKR:change: was self.
    main_window = g.app.gui.main_window
    # EKR:change.
        # QtWidgets.QMainWindow.__init__(self, parent)

    main_window._closeflag = 0  # Used during closing/restarting

    # EKR:change.
        # # Init window title and application icon
        # self.setMainTitle()
    
    # EKR:change.
    main.loadAppIcons()
    pyzo.icon = g.app.gui.appIcon
    # Don't patch this now. It might be a good indicator.
    # pyzo.iconRunning = g.app.gui.appIcon
    
        # loadAppIcons()
    # EKR:change.
        # self.setWindowIcon(pyzo.icon)
    # EKR:change.
        # Restore window geometry.
        # self.resize(800, 600) # default size
        # self.restoreGeometry()
    # EKR:change.
        # Show splash screen (we need to set our color too)
        # w = SplashWidget(self, distro='no distro')
    # EKR:change.
        # self.setCentralWidget(w)
    # EKR:change.
       #  self.setStyleSheet("QMainWindow { background-color: #268bd2;}")

    # Show empty window and disable updates for a while

    # EKR:change.
        # self.show()
        # self.paintNow()
        # self.setUpdatesEnabled(False)
    # EKR:change.
        # Determine timeout for showing splash screen
        # splash_timeout = time.time() + 1.0
    # EKR:change.
        # Set locale of main widget, so that qt strings are translated
        # in the right way
        # if locale:
            # self.setLocale(locale)
  
    # Set pyzo.main.
    pyzo.main = main_window
    
    # EKR:change-Add do-nothing methods.
    pyzo.main.setMainTitle = g.TracingNullObject(tag='pyzo.main.setMainTitle()')
    pyzo.main.restart = g.TracingNullObject(tag='pyzo.main.restart()')

    # Init dockwidget settings
    main_window.setTabPosition(QtCore.Qt.AllDockWidgetAreas,QtWidgets.QTabWidget.South)
    main_window.setDockOptions(
        QtWidgets.QMainWindow.AllowNestedDocks |
        QtWidgets.QMainWindow.AllowTabbedDocks
        #|  QtWidgets.QMainWindow.AnimatedDocks
    )

    # Set window atrributes
    main_window.setAttribute(QtCore.Qt.WA_AlwaysShowToolTips, True)

    # EKR:change.
    # Load icons and fonts
    main.loadIcons()
    main.loadFonts()
        # loadIcons()
        # loadFonts()

    # EKR:change.
        # # Set qt style and test success
        # self.setQtStyle(None) # None means init!
    # EKR:change.
        # # Hold the splash screen if needed
        # while time.time() < splash_timeout:
            # QtWidgets.qApp.flush()
            # QtWidgets.qApp.processEvents()
            # time.sleep(0.05)
    # EKR:change.
    # Populate the window (imports more code)
    self.main_window_populate()
        # self._populate()
        
    # EKR:change: new code.
    self.load_all_pyzo_docks()

    # EKR:change.
    # Revert to normal background, and enable updates
    main_window.setStyleSheet('')
    main_window.setUpdatesEnabled(True)

    # EKR:change. Could this be a problem?
        # # Restore window state, force updating, and restore again
        # self.restoreState()
        # self.paintNow()
        # self.restoreState()

    # EKR:change.
        # Present user with wizard if he/she is new.
        # if pyzo.config.state.newUser:
            # from pyzo.util.pyzowizard import PyzoWizard
            # w = PyzoWizard(self)
            # w.show() # Use show() instead of exec_() so the user can interact with pyzo

    # EKR:change
        # # Create new shell config if there is None
        # if not pyzo.config.shellConfigs2:
            # from pyzo.core.kernelbroker import KernelInfo
            # pyzo.config.shellConfigs2.append( KernelInfo() )
    from pyzo.core.kernelbroker import KernelInfo
        # pyzo.config.shellConfigs2.append( KernelInfo() )
    pyzo.config.shellConfigs2 = [KernelInfo()]

    # EKR:change Set background.
        # bg = getattr(pyzo.config.settings, 'dark_background', '#657b83')
            # # Default: solarized base00
        # try:
            # self.setStyleSheet(f"background: {bg}") 
        # except Exception:
            # g.es_exception()

    # Focus on editor
    e = pyzo.editors.getCurrentEditor()
    if e is not None:
        e.setFocus()

    # Handle any actions
    commandline.handle_cmd_args()
    
    # print('END main_window_ctor\n')
#@+node:ekr.20190816132847.1: *5* pz.main_window_populate
def main_window_populate(self):
    """
    Simulate MainWindow._populate().
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    # EKR:change: replaces self in most places.
    main_window = g.app.gui.main_window

    # print('\nBEGIN main_window_populate\n')
    
    # EKR:change-new imports
    from pyzo.core.main import callLater

    # Delayed imports
    from pyzo.core.editorTabs import EditorTabs
    from pyzo.core.shellStack import ShellStackWidget
    from pyzo.core import codeparser
    from pyzo.core.history import CommandHistory
    from pyzo.tools import ToolManager

    # Instantiate tool manager
    pyzo.toolManager = ToolManager()

    # EKR: Disabled in original.
        # Check to install conda now ...
        # from pyzo.util.bootstrapconda import check_for_conda_env
        # check_for_conda_env()

    # Instantiate and start source-code parser
    if pyzo.parser is None:
        pyzo.parser = codeparser.Parser()
        pyzo.parser.start()

    # Create editor stack and make the central widget
    # EKR:change. Use None, not self.
    pyzo.editors = EditorTabs(None)
    
    # EKR:change. Create an Editors dock.
    self.make_global_dock('Editors', pyzo.editors)
        # self.setCentralWidget(pyzo.editors)

    # Create floater for shell
    # EKR:change: use a global *Leo* dock
    dock = g.app.gui.create_dock_widget(
        closeable=True,
        moveable=True,
        height=50,
        name='Shells',
    )
    # Old code
        # self._shellDock = dock = QtWidgets.QDockWidget(self)
        # if pyzo.config.settings.allowFloatingShell:
            # dock.setFeatures(dock.DockWidgetMovable | dock.DockWidgetFloatable)
        # else:
            # dock.setFeatures(dock.DockWidgetMovable)
    dock.setObjectName('shells')
        # dock.setWindowTitle('Shells')
    
    # EKR:change: Make the dock a *global* dock.
    main_window.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
        # self.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)

    # Create shell stack
    # EKR:change. Use None, not self.
    
    # A hack: patch _get_interpreters_win
    if 1:
        import pyzo.util.interpreters as interps
        interps._get_interpreters_win = _get_interpreters_win

    pyzo.shells = ShellStackWidget(None)
    dock.setWidget(pyzo.shells)

    # Initialize command history
    pyzo.command_history = CommandHistory('command_history.py')

    # Create the default shell when returning to the event queue
    callLater(pyzo.shells.addShell)

    # EKR:change.
    pyzo.status = None
    # Create statusbar
        # if pyzo.config.view.showStatusbar:
            # pyzo.status = self.statusBar()
        # else:
            # pyzo.status = None
            # self.setStatusBar(None)
            
    from pyzo.core import menu
    pyzo.keyMapper = menu.KeyMapper()
    
    # EKR:change: Monkey-patch pyzo.keyMapper.setShortcut.
    g.funcToMethod(setShortcut, pyzo.keyMapper.__class__)
    
    # EKR-change: init_pyzo_menu does this later.
        # # Add the context menu to the editor
        # pyzo.editors.addContextMenu()
        # pyzo.shells.addContextMenu()
            
    # print('END main_window_populate\n')
#@+node:ekr.20190813161921.1: *5* pz.make_global_dock
def make_global_dock(self, name, widget):
    """Create a dock with the given name and widget in the global main window."""
    main_window = g.app.gui.main_window
    dock = g.app.gui.create_dock_widget(
        closeable=True,
        moveable=True, # Implies floatable.
        height=100,
        name=name,
    )
    dock.setWidget(widget)
    area = QtCore.Qt.LeftDockWidgetArea
    main_window.addDockWidget(area, dock)
    widget.show()
#@+node:ekr.20190816131343.1: *5* pz.pyzo_start
def pyzo_start(self):
    """
    A copy of pyzo.start, adapted for Leo.
    
    Called at start2 time.  c is not available.
    
    This code is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    
    # Do some imports
    from pyzo.core import pyzoLogging  # to start logging asap
        # EKK: All print statements after this will appear in the Logger dock.
        # Unless we change pyzoLogging itself, this import will happen soon anyway.
    assert pyzoLogging

    # print('\nBEGIN pyzo_start\n')
    
    # EKR:change.
    # from pyzo.core.main import MainWindow

    # Apply users' preferences w.r.t. date representation etc
    for x in ('', 'C', 'en_US', 'en_US.utf8', 'en_US.UTF-8'):
        try:
            locale.setlocale(locale.LC_ALL, x)
            break
        except locale.Error:
            pass

    # Set to be aware of the systems native colors, fonts, etc.
    QtWidgets.QApplication.setDesktopSettingsAware(True)

    # EKR-change: the only remaining code from my_app_ctor.
    sys.argv = sys.argv[:1]
        # Instantiate the application.
        # QtWidgets.qApp = MyApp(sys.argv)
        # my_app_ctor(sys.argv)

    # EKR:change.
        # # Choose language, get locale
        # appLocale = setLanguage(config.settings.language)
    # EKR:change.
    # Create main window, using the selected locale
        # MainWindow(None, appLocale)
    self.main_window_ctor()

    # EKR:change.
        # Enter the main loop
        # QtWidgets.qApp.exec_()

    # print('END pyzo_start\n')
#@+node:ekr.20190506094028.1: ** Demo stuff
#@+node:ekr.20190506094028.2: *3* @@button demo1 @key=Ctrl-8
g.cls()
if c.isChanged(): c.save()
import imp
import leo.plugins.demo as demo
imp.reload(demo)
<< class MyDemo >>
h = 'demo1-commands'
button_p = g.findNodeAnywhere(c, '@button demo1 @key=Ctrl-8')
commands = g.findNodeInTree(c, button_p, h)
if commands:
    MyDemo(c).start(commands)
else:
    print('not found', h, c.p.h)
#@+node:ekr.20190506094028.3: *4* << class MyDemo >>
class MyDemo(demo.Demo):
    
    @others
    
#@+node:ekr.20190506094028.4: *5* setup
def setup(self, p):
    
    c = self.c
    self.delta = 0
    self.clear_log()
    p = g.findNodeAnywhere(c, 'Demo area')
    if p:
        c.selectPosition(p)
#@+node:ekr.20190506094028.5: *5* teardown
def teardown(self):
    
    c = self.c
    if self.delta:
        self.set_text_delta(-self.delta)
    p = g.findNodeAnywhere(c, 'Demo area')
    if p:
        c.selectPosition(p)
        next = p.next()
        if next and next.h == 'This is a test':
            c.selectPosition(next)
            next.doDelete()
            c.selectPosition(p)
            c.setChanged(False)
            c.redraw()
    

#@+node:ekr.20190506094028.6: *4* demo1-commands
print('demo1-commands')
# c.contractAllHeadlines()
#@+node:ekr.20190506094028.7: *5* @ignore-tree
#@+node:ekr.20190506094028.8: *6* set_text_delta
print('increasing text size by 10')
demo.delta = 10
demo.set_text_delta(demo.delta)
#@+node:ekr.20190506094028.9: *6* undo
undo_type = c.undoer.undoType
if undo_type == 'Insert Node':
    c.undoer.undo()
#@+node:ekr.20190506094028.10: *6* caption
demo.caption('My Caption', 'body')
#@+node:ekr.20190506094028.11: *6* @image
demo.delete_widgets()
fn = 'SplashScreen.ico'
demo.image('body', fn, center=True, height=None, width=None)
#@+node:ekr.20190506094028.12: *6* open menu
demo.delete_widgets()
demo.open_menu('Import')
#@+node:ekr.20190506094028.13: *6* close menu
demo.dismiss_menu_bar()
#@+node:ekr.20190506094028.14: *6* Alt-X insert-node
demo.key('Alt+x') # Not the same as Alt-X
demo.keys('insert-node')
# demo.wait(0.8)
# demo.key('\n') # Works.
#@+node:ekr.20190506094028.15: *6* Return
demo.key('\n')
#@+node:ekr.20190506094028.16: *5* headline
c.k.simulateCommand('insert-node')
demo.head_keys('This is a test')
#@+node:ekr.20190506094028.17: *3* Test: import c:\test\demo-it.el
g.cls()
import imp
import leo.plugins.importers.linescanner as linescanner
import leo.plugins.importers.elisp as elisp
imp.reload(linescanner)
imp.reload(elisp)
x = elisp.Elisp_Importer(c.importCommands, atAuto=False)
with open('c:/test/demo-it.el') as f:
    s = f.read()
parent = p.next()
assert parent.h == 'demo.el', parent.h
parent.b = ''
parent.deleteAllChildren()
try:
    x.run(s, parent)
except Exception:
    g.es_exception()
parent.expand()
c.selectPosition(parent)
c.redraw()
# g.printList(g.splitLines(s))
#@+node:ekr.20190506094028.18: *3* demo.image & helper
def image(self, fn, center=None, height=None, pane=None, width=None):
    '''Put an image in the indicated pane.'''
    parent = self.pane_widget(pane or 'body')
    if parent:
        w = QtWidgets.QLabel('label', parent)
        fn = self.resolve_icon_fn(fn)
        if not fn: return None
        pixmap = QtGui.QPixmap(fn)
        if not pixmap:
            return g.trace('Not a pixmap: %s' % (fn))
        if height:
            pixmap = pixmap.scaledToHeight(height)
        if width:
            pixmap = pixmap.scaledToWidth(width)
        w.setPixmap(pixmap)
        if center:
            g_w = w.geometry()
            g_p = parent.geometry()
            dx = (g_p.width() - g_w.width()) / 2
            w.move(g_w.x() + dx, g_w.y() + 10)
        w.show()
        self.widgets.append(w)
        return w
    else:
        g.trace('bad pane: %s' % (pane))
        return None
#@+node:ekr.20190506094028.19: *4* demo.resolve_icon_fn
def resolve_icon_fn(self, fn):
    '''Resolve fn relative to the Icons directory.'''
    dir_ = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons')
    path = g.os_path_finalize_join(dir_, fn)
    if g.os_path_exists(path):
        return path
    else:
        g.trace('does not exist: %s' % (path))
        return None
#@+node:ekr.20190506094028.20: *3* demo.caption & body, log, tree
def caption(self, s, pane): # To do: center option.
    '''Pop up a QPlainTextEdit in the indicated pane.'''
    parent = self.pane_widget(pane)
    if parent:
        s = s.rstrip()
        if s and s[-1].isalpha(): s = s + '.'
        w = QtWidgets.QPlainTextEdit(s, parent)
        w.setObjectName('screencastcaption')
        self.widgets.append(w)
        w2 = self.pane_widget(pane)
        geom = w2.geometry()
        w.resize(geom.width(), min(150, geom.height() / 2))
        off = QtCore.Qt.ScrollBarAlwaysOff
        w.setHorizontalScrollBarPolicy(off)
        w.setVerticalScrollBarPolicy(off)
        w.show()
        return w
    else:
        g.trace('bad pane: %s' % (pane))
        return None

def body(self, s):
    return self.caption(s, 'body')

def log(self, s):
    return self.caption(s, 'log')

def tree(self, s):
    return self.caption(s, 'tree')
#@+node:ekr.20190506094028.21: *3* demo.body, log, tree
def body(self, s):
    return TextEdit(s, 'body')

def log(self, s):
    return TextEdit(s, 'log')

def tree(self, s):
    return TextEdit(s, 'tree')
#@+node:ekr.20190506094028.22: *3* Demo area
@language python

# A python comment.
#@+node:ekr.20210509173443.1: ** Documentation
#@+node:ekr.20210509173440.1: *3* Leo and ZODB
#@+node:ekr.20210509173440.2: *4* @rst html\zodb.html
###################
Using ZODB with Leo
###################

.. _`ZODB`: http://www.zope.org/Wikis/ZODB/guide/zodb.html

This chapter discusses how to write Leo scripts that store and retrieve data using `ZODB`_.

.. contents:: Contents
    :depth: 2
    :local:

#@+node:ekr.20210509173440.3: *5* Configuring Leo to use zodb
.. _`Installing ZODB`: http://www.zope.org/Wikis/ZODB/guide/node3.html#SECTION000310000000000000000

To enable zodb scripting within Leo, you must set use_zodb = True in the root node of leoNodes.py. You must also install ZODB itself.  See `Installing ZODB`_ for details.

When ZODB is installed and use_zodb is True, Leo's vnode class becomes a subclass of ZODB.Persistence.Persistent. This is all that is needed to save/retrieve vnodes or tnodes to/from the ZODB.

**Important notes**:

- Scripts **should not** store or retrieve positions using the ZODB! Doing so makes sense neither from Leo's point of view nor from ZODB's point of view.

- The examples below show how to store or retrieve Leo data by accessing the so-called root of a ZODB connection. However, these are only examples. Scripts are free to do with Leo's vnodes *anything* that can be done with ZODB.Persistence.Persistent objects.
#@+node:ekr.20210509173440.4: *5* Initing zodb
Scripts should call g.init_zodb to open a ZODB.Storage file. g.init_zodb returns an instance of ZODB.DB.  For example::

    db = g.init_zodb (zodbStorageFileName)

You can call g.init_zodb as many times as you like. Only the first call for any path actually does anything: subsequent calls for a previously opened path simply return the same value as the first call.
#@+node:ekr.20210509173440.5: *5* Writing data to zodb
The following script writes v, a tree of vnodes, to zodb::

    db = g.init_zodb (zodbStorageFileName)
    connection = db.open()
    try:
        root = connection.root()
        root[aKey] = v # See next section for how to define aKey.
    finally:
        get_transaction().commit()
        connection.close()

Notes:

- v must be a vnode.
  Scripts should *not* attempt to store Leo positions in the zodb.
  v can be the root of an entire outline or a subtree.
  For example, either of the following would be reasonable::

    root[aKey] = c.rootPosition().v
    root[aKey] = c.p.v

- To write a single vnode without writing any of its children you can use v.detach.
  For example::

    root[aKey] = v.detach()

- **Important**: It is simplest if only one zodb connection is open at any one time,
  so scripts would typically close the zodb connection immediately after processing the data.
  The correct way to do this is in a finally statement, as shown above.

- The script above does not define aKey.
  The following section discusses how to define reasonable zodb keys.
#@+node:ekr.20210509173440.6: *5* Defining zodb keys
The keys used to store and retrieve data in connection.root() can be any string that uniquely identifies the data. The following are only suggestions; you are free to use any string you like.

1. When saving a file, you would probably use a key that is similar to a real file path.
   For example::

        aKey = c.fileName()

2. When saving a single vnode or tree of vnodes, say v,
   a good choice would be to use v's gnx, namely::

        aKey = g.app.nodeIndices.toString(v.fileIndex)

   Note that v.detach() does not automatically copy v.fileIndex to the detached node,
   so when writing a detached node you would typically do the following::

       v2 = v.detach()
       v2.fileIndex = v.fileIndex
       aKey = g.app.nodeIndices.toString(v2.fileIndex)
#@+node:ekr.20210509173440.7: *5* Reading data from zodb
The following script reads a tree of vnodes from zodb and sets p as the root position of the tree::

    try:
        connection = db.open()
        root = connection.root()
        v = root.get(aKey)
        p = leoNodes.position(v)
    finally:
        get_transaction().commit()
        connection.close()
#@+node:ekr.20210509173440.8: *5* About connections
The scripts shown above close the zodb connection after processing the data. This is by far the simplest strategy. I recommend it for typical scripts.

**Important**: you must **leave the connection open** if your script modifies persistent data in any way. (Actually, this statement is not really true, but you must define zodb transaction managers if you intend to use multiple connections simultaneously. This complication is beyond the scope of this documentation.) For example, it would be possible to create a new Leo outline from the data just read, but the script must leave the connection open. I do not recommend this tactic, but for the adventurous here is some sample code:

.. code-block:: python

    connection = self.db.open()
    root = connection.root()
    v = root.get(fileName)
    if v:
        c2 = c.new()
        c2.openDirectory = c.openDirectory # A hack.
        c2.mFileName = fileName # Another hack.
        c2.beginUpdate()
        try:
            c2.setRootVnode(v)
            c2Root = c2.rootPosition()
            c2.atFileCommands.readAll(c2Root)
            g.es_print('zodb read: %s' % (fileName))
        finally:
            c2.endUpdate()
        # Do *not* close the connection while the new Leo window is open!
    else:
        g.es_print('zodb read: not found: %s' % (fileName))


This will work **provided** that no other zodb connection is ever opened while this connection is opened. Unless special zodb precautions are taken (like defining zodb transaction managers) calling get_transaction().commit() will affect **all** open connections. You have been warned.
#@+node:ekr.20210509173440.9: *5* Convenience routines
#@+node:ekr.20210509173440.10: *6* g.init_zodb (pathToZodbStorage,verbose=True)
This function inits the zodb. pathToZodbStorage is the full path to the zodb storage file. You can call g.init_zodb as many times as you like. Only the first call for any path actually does anything: subsequent calls for a previously opened path simply return the same value as the first call.
#@+node:ekr.20210509173440.11: *6* v.detach()
This vnode method returns v2, a copy of v that is completely detached from the outline. v2.fileIndex is unrelated to v.fileIndex initially, but it may be convenient to copy this field::

    v2 = v.detach()
    v2.fileIndex = v.fileIndex
#@+node:ekr.20210311074039.1: ** Find patterns
#@+node:ekr.20190122185223.1: *3*  regex: find all kwargs
@language text

Find all kwargs using regex:

^\s*(def\s+\w+\s*\(.*\=.*\))

found 1154 nodes

^\s*(def\s+\w+\s*\(.*\=.*\=.*\))

found 284 nodes
#@+node:ekr.20191229062845.1: ** From leoAst.py
#@+node:ekr.20201017155356.1: *3*  ----- Support for Python 3.8 type nodes
@language rest
@wrap

For more details, see:
https://github.com/leo-editor/leo-editor/issues/

I'll support these nodes only if someone complains.

@language python
#@+node:ekr.20201017072904.1: *4* tog: Type Expressions (Python 3.8+) (not used)
# Type expressions are *comments*.
#@+node:ekr.20201017072929.1: *5* tog.FunctionType
# FunctionType(expr* argtypes, expr returns)

def do_FunctionType(self, node):

    argtypes = getattr(node, 'argtypes', [])
    for z in argtypes:
        yield from self.gen(z)
    yield from self.gen(node.returns)
#@+node:ekr.20201017072932.1: *5* tog.NamedExpr
# NamedExpr(expr target, expr value)

def do_NamedExpr(self, node):
    
    yield from self.gen(node.target)
    yield from self.gen(node.value)
#@+node:ekr.20201017072932.2: *5* tog.TypeIgnore
# type_ignore = TypeIgnore(int lineno, string tag)

def do_TypeIgnore(self, node):
    
    yield from self.gen(node.lineno)
    yield from self.gen(node.tag)
#@+node:ekr.20200729081252.1: *3* FullTraverser classes
#@+node:ekr.20141012064706.18399: *4* class AstFormatter
class AstFormatter:
    """
    A class to recreate source code from an AST.

    This does not have to be perfect, but it should be close.

    Also supports optional annotations such as line numbers, file names, etc.
    """
    # No ctor.
    # pylint: disable=consider-using-enumerate

    in_expr = False
    level = 0

    @others
#@+node:ekr.20141012064706.18402: *5* f.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    self.level = level
    val = self.visit(node, *args, **keys)
    return val.rstrip() if val else ''
#@+node:ekr.20141012064706.18403: *5* f.visit
def visit(self, node, *args, **keys):
    """Return the formatted version of an Ast node, or list of Ast nodes."""

    if isinstance(node, (list, tuple)):
        return ','.join([self.visit(z) for z in node])
    if node is None:
        return 'None'
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    s = method(node, *args, **keys)
    assert isinstance(s, str), type(s)
    return s
#@+node:ekr.20141012064706.18469: *5* f.indent
def indent(self, s):
    return f'%s%s' % (' ' * 4 * self.level, s)
#@+node:ekr.20141012064706.18404: *5* f: Contexts
#@+node:ekr.20141012064706.18405: *6* f.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, print_body=True):

    result = []
    name = node.name  # Only a plain string is valid.
    bases = [self.visit(z) for z in node.bases] if node.bases else []
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            bases.append(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
    if getattr(node, 'starargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.starargs))
    if getattr(node, 'kwargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.kwargs))
    if bases:
        result.append(self.indent(f'class %s(%s):\n' % (name, ','.join(bases))))
    else:
        result.append(self.indent(f'class %s:\n' % name))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18406: *6* f.FunctionDef & AsyncFunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False, print_body=True):
    """Format a FunctionDef node."""
    result = []
    if node.decorator_list:
        for z in node.decorator_list:
            result.append(f'@%s\n' % self.visit(z))
    name = node.name  # Only a plain string is valid.
    args = self.visit(node.args) if node.args else ''
    asynch_prefix = 'asynch ' if async_flag else ''
    if getattr(node, 'returns', None):  # Python 3.
        returns = self.visit(node.returns)
        result.append(self.indent(f'%sdef %s(%s): -> %s\n' % (
            asynch_prefix, name, args, returns)))
    else:
        result.append(self.indent(f'%sdef %s(%s):\n' % (
            asynch_prefix, name, args)))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFunctionDef(self, node):
    return self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20141012064706.18407: *6* f.Interactive
def do_Interactive(self, node):
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18408: *6* f.Module
def do_Module(self, node):
    assert 'body' in node._fields
    result = ''.join([self.visit(z) for z in node.body])
    return result
#@+node:ekr.20141012064706.18409: *6* f.Lambda
def do_Lambda(self, node):
    return self.indent(f'lambda %s: %s' % (
        self.visit(node.args),
        self.visit(node.body)))
#@+node:ekr.20141012064706.18410: *5* f: Expressions
#@+node:ekr.20141012064706.18411: *6* f.Expr
def do_Expr(self, node):
    """An outer expression: must be indented."""
    assert not self.in_expr
    self.in_expr = True
    value = self.visit(node.value)
    self.in_expr = False
    return self.indent(f'%s\n' % value)
#@+node:ekr.20141012064706.18412: *6* f.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s\n' % self.visit(node.body)
#@+node:ekr.20141012064706.18413: *6* f.GeneratorExp
def do_GeneratorExp(self, node):
    elt = self.visit(node.elt) or ''
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'<gen %s for %s>' % (elt, ','.join(gens))
#@+node:ekr.20141012064706.18414: *6* f.ctx nodes
def do_AugLoad(self, node):
    return 'AugLoad'

def do_Del(self, node):
    return 'Del'

def do_Load(self, node):
    return 'Load'

def do_Param(self, node):
    return 'Param'

def do_Store(self, node):
    return 'Store'
#@+node:ekr.20141012064706.18415: *5* f: Operands
#@+node:ekr.20141012064706.18416: *6* f.arguments
# 2: arguments = (expr* args, identifier? vararg, identifier?
#                arg? kwarg, expr* defaults)
# 3: arguments = (arg*  args, arg? vararg,
#                arg* kwonlyargs, expr* kw_defaults,
#                arg? kwarg, expr* defaults)

def do_arguments(self, node):
    """Format the arguments node."""
    kind = node.__class__.__name__
    assert kind == 'arguments', kind
    args = [self.visit(z) for z in node.args]
    defaults = [self.visit(z) for z in node.defaults]
    args2 = []
    n_plain = len(args) - len(defaults)
    for i in range(len(node.args)):
        if i < n_plain:
            args2.append(args[i])
        else:
            args2.append(f'%s=%s' % (args[i], defaults[i-n_plain]))
    # Add the vararg and kwarg expressions.
    vararg = getattr(node, 'vararg', None)
    if vararg: args2.append('*'+self.visit(vararg))
    kwarg = getattr(node, 'kwarg', None)
    if kwarg: args2.append(f'**'+self.visit(kwarg))
    return ','.join(args2)
#@+node:ekr.20141012064706.18417: *6* f.arg (Python3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        return self.visit(node.annotation)
    return node.arg
#@+node:ekr.20141012064706.18418: *6* f.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    return f'%s.%s' % (
        self.visit(node.value),
        node.attr)  # Don't visit node.attr: it is always a string.
#@+node:ekr.20141012064706.18419: *6* f.Bytes
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20141012064706.18420: *6* f.Call & f.keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    func = self.visit(node.func)
    args = [self.visit(z) for z in node.args]
    for z in node.keywords:
        # Calls f.do_keyword.
        args.append(self.visit(z))
    if getattr(node, 'starargs', None):
        args.append(f'*%s' % (self.visit(node.starargs)))
    if getattr(node, 'kwargs', None):
        args.append(f'**%s' % (self.visit(node.kwargs)))
    args = [z for z in args if z]  # Kludge: Defensive coding.
    s = f'%s(%s)' % (func, ','.join(args))
    return s if self.in_expr else self.indent(s+'\n')
        # 2017/12/15.
#@+node:ekr.20141012064706.18421: *7* f.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    value = self.visit(node.value)
    # This is a keyword *arg*, not a Python keyword!
    return f'%s=%s' % (node.arg, value)
#@+node:ekr.20141012064706.18422: *6* f.comprehension
def do_comprehension(self, node):
    result = []
    name = self.visit(node.target)  # A name.
    it = self.visit(node.iter)  # An attribute.
    result.append(f'%s in %s' % (name, it))
    ifs = [self.visit(z) for z in node.ifs]
    if ifs:
        result.append(f' if %s' % (''.join(ifs)))
    return ''.join(result)
#@+node:ekr.20170721073056.1: *6* f.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20141012064706.18423: *6* f.Dict
def do_Dict(self, node):
    result = []
    keys = [self.visit(z) for z in node.keys]
    values = [self.visit(z) for z in node.values]
    if len(keys) == len(values):
        result.append('{\n' if keys else '{')
        items = []
        for i in range(len(keys)):
            items.append(f'  %s:%s' % (keys[i], values[i]))
        result.append(',\n'.join(items))
        result.append('\n}' if keys else '}')
    else:
        print(
            f"Error: f.Dict: len(keys) != len(values)\n"
            f"keys: {repr(keys)}\nvals: {repr(values)}")
    return ''.join(result)
#@+node:ekr.20160523101618.1: *6* f.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    key = self.visit(node.key)
    value = self.visit(node.value)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s:%s for %s' % (key, value, ''.join(gens))
#@+node:ekr.20141012064706.18424: *6* f.Ellipsis
def do_Ellipsis(self, node):
    return '...'
#@+node:ekr.20141012064706.18425: *6* f.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20170721075130.1: *6* f.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    return f'%s%s%s' % (
        self.visit(node.value),
        self.visit(node.conversion) if node.conversion else '',
        self.visit(node.format_spec) if node.format_spec else '')
#@+node:ekr.20141012064706.18426: *6* f.Index
def do_Index(self, node):
    return self.visit(node.value)
#@+node:ekr.20170721080559.1: *6* f.JoinedStr (Python 3.6)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):

    if node.values:
        for value in node.values:
            self.visit(value)
#@+node:ekr.20141012064706.18427: *6* f.List
def do_List(self, node):
    # Not used: list context.
    # self.visit(node.ctx)
    elts = [self.visit(z) for z in node.elts]
    elts = [z for z in elts if z]  # Defensive.
    return f'[%s]' % ','.join(elts)
#@+node:ekr.20141012064706.18428: *6* f.ListComp
def do_ListComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18429: *6* f.Name & NameConstant
def do_Name(self, node):
    return node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return s
#@+node:ekr.20141012064706.18430: *6* f.Num
def do_Num(self, node):
    return repr(node.n)
#@+node:ekr.20141012064706.18431: *6* f.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20160523101929.1: *6* f.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523102226.1: *6* f.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18432: *6* f.Slice
def do_Slice(self, node):
    lower, upper, step = '', '', ''
    if getattr(node, 'lower', None) is not None:
        lower = self.visit(node.lower)
    if getattr(node, 'upper', None) is not None:
        upper = self.visit(node.upper)
    if getattr(node, 'step', None) is not None:
        step = self.visit(node.step)
    if step:
        return f'%s:%s:%s' % (lower, upper, step)
    return f'%s:%s' % (lower, upper)
#@+node:ekr.20141012064706.18433: *6* f.Str
def do_Str(self, node):
    """This represents a string constant."""
    return repr(node.s)
#@+node:ekr.20141012064706.18434: *6* f.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    value = self.visit(node.value)
    the_slice = self.visit(node.slice)
    return f'%s[%s]' % (value, the_slice)
#@+node:ekr.20141012064706.18435: *6* f.Tuple
def do_Tuple(self, node):
    elts = [self.visit(z) for z in node.elts]
    return f'(%s)' % ','.join(elts)
#@+node:ekr.20141012064706.18436: *5* f: Operators
#@+node:ekr.20141012064706.18437: *6* f.BinOp
def do_BinOp(self, node):
    return f'%s%s%s' % (
        self.visit(node.left),
        op_name(node.op),
        self.visit(node.right))
#@+node:ekr.20141012064706.18438: *6* f.BoolOp
def do_BoolOp(self, node):
    op_name_ = op_name(node.op)
    values = [self.visit(z).strip() for z in node.values]
    return op_name_.join(values)
#@+node:ekr.20141012064706.18439: *6* f.Compare
def do_Compare(self, node):
    result = []
    lt = self.visit(node.left)
    # ops   = [self.visit(z) for z in node.ops]
    ops = [op_name(z) for z in node.ops]
    comps = [self.visit(z) for z in node.comparators]
    result.append(lt)
    assert len(ops) == len(comps), repr(node)
    for i in range(len(ops)):
        result.append(f'%s%s' % (ops[i], comps[i]))
    return ''.join(result)
#@+node:ekr.20141012064706.18440: *6* f.UnaryOp
def do_UnaryOp(self, node):
    return f'%s%s' % (
        op_name(node.op),
        self.visit(node.operand))
#@+node:ekr.20141012064706.18441: *6* f.ifExp (ternary operator)
def do_IfExp(self, node):
    return f'%s if %s else %s ' % (
        self.visit(node.body),
        self.visit(node.test),
        self.visit(node.orelse))
#@+node:ekr.20141012064706.18442: *5* f: Statements
#@+node:ekr.20170721074105.1: *6* f.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    return self.indent(f'%s:%s=%s\n' % (
        self.visit(node.target),
        self.visit(node.annotation),
        self.visit(node.value),
    ))
#@+node:ekr.20141012064706.18443: *6* f.Assert
def do_Assert(self, node):
    test = self.visit(node.test)
    if getattr(node, 'msg', None):
        message = self.visit(node.msg)
        return self.indent(f'assert %s, %s' % (test, message))
    return self.indent(f'assert %s' % test)
#@+node:ekr.20141012064706.18444: *6* f.Assign
def do_Assign(self, node):
    return self.indent(f'%s=%s\n' % (
        '='.join([self.visit(z) for z in node.targets]),
        self.visit(node.value)))
#@+node:ekr.20141012064706.18445: *6* f.AugAssign
def do_AugAssign(self, node):
    return self.indent(f'%s%s=%s\n' % (
        self.visit(node.target),
        op_name(node.op),  # Bug fix: 2013/03/08.
        self.visit(node.value)))
#@+node:ekr.20160523100504.1: *6* f.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    return self.indent(f'await %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18446: *6* f.Break
def do_Break(self, node):
    return self.indent(f'break\n')
#@+node:ekr.20141012064706.18447: *6* f.Continue
def do_Continue(self, node):
    return self.indent(f'continue\n')
#@+node:ekr.20141012064706.18448: *6* f.Delete
def do_Delete(self, node):
    targets = [self.visit(z) for z in node.targets]
    return self.indent(f'del %s\n' % ','.join(targets))
#@+node:ekr.20141012064706.18449: *6* f.ExceptHandler
def do_ExceptHandler(self, node):
    
    result = []
    result.append(self.indent('except'))
    if getattr(node, 'type', None):
        result.append(f' %s' % self.visit(node.type))
    if getattr(node, 'name', None):
        if isinstance(node.name, ast.AST):
            result.append(f' as %s' % self.visit(node.name))
        else:
            result.append(f' as %s' % node.name)  # Python 3.x.
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18450: *6* f.Exec
# Python 2.x only

def do_Exec(self, node):
    body = self.visit(node.body)
    args = []  # Globals before locals.
    if getattr(node, 'globals', None):
        args.append(self.visit(node.globals))
    if getattr(node, 'locals', None):
        args.append(self.visit(node.locals))
    if args:
        return self.indent(f'exec %s in %s\n' % (
            body, ','.join(args)))
    return self.indent(f'exec {body}\n')
#@+node:ekr.20141012064706.18451: *6* f.For & AsnchFor (Python 3)
def do_For(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%sfor %s in %s:\n' % (
        'async ' if async_flag else '',
        self.visit(node.target),
        self.visit(node.iter))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFor(self, node):
    return self.do_For(node, async_flag=True)
#@+node:ekr.20141012064706.18452: *6* f.Global
def do_Global(self, node):
    return self.indent(f'global %s\n' % (
        ','.join(node.names)))
#@+node:ekr.20141012064706.18453: *6* f.If
def do_If(self, node):
    result = []
    result.append(self.indent(f'if %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent(f'else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18454: *6* f.Import & helper
def do_Import(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'import %s\n' % (
        ','.join(names)))
#@+node:ekr.20141012064706.18455: *7* f.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert ast2.__class__.__name__ == 'alias', (repr(ast2))
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20141012064706.18456: *6* f.ImportFrom
def do_ImportFrom(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'from %s import %s\n' % (
        node.module,
        ','.join(names)))
#@+node:ekr.20160317050557.2: *6* f.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    return self.indent(f'nonlocal %s\n' % ', '.join(node.names))
#@+node:ekr.20141012064706.18457: *6* f.Pass
def do_Pass(self, node):
    return self.indent('pass\n')
#@+node:ekr.20141012064706.18458: *6* f.Print
# Python 2.x only

def do_Print(self, node):
    vals = []
    for z in node.values:
        vals.append(self.visit(z))
    if getattr(node, 'dest', None):
        vals.append(f'dest=%s' % self.visit(node.dest))
    if getattr(node, 'nl', None):
        # vals.append('nl=%s' % self.visit(node.nl))
        vals.append(f'nl=%s' % node.nl)
    return self.indent(f'print(%s)\n' % (
        ','.join(vals)))
#@+node:ekr.20141012064706.18459: *6* f.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):
    args = []
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            args.append(self.visit(getattr(node, attr)))
    if args:
        return self.indent(f'raise %s\n' % (
            ','.join(args)))
    return self.indent('raise\n')
#@+node:ekr.20141012064706.18460: *6* f.Return
def do_Return(self, node):
    if node.value:
        return self.indent(f'return %s\n' % (
            self.visit(node.value)))
    return self.indent('return\n')
#@+node:ekr.20160317050557.3: *6* f.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    return '*' + self.visit(node.value)
#@+node:ekr.20141012064706.18461: *6* f.Suite
# def do_Suite(self,node):
    # for z in node.body:
        # s = self.visit(z)
#@+node:ekr.20160317050557.4: *6* f.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):  # Python 3

    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    if node.finalbody:
        result.append(self.indent('finally:\n'))
        for z in node.finalbody:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18462: *6* f.TryExcept
def do_TryExcept(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18463: *6* f.TryFinally
def do_TryFinally(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append(self.indent('finally:\n'))
    for z in node.finalbody:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18464: *6* f.While
def do_While(self, node):
    result = []
    result.append(self.indent(f'while %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18465: *6* f.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%swith ' % ('async ' if async_flag else '')))
    if getattr(node, 'context_expression', None):
        result.append(self.visit(node.context_expresssion))
    vars_list = []
    if getattr(node, 'optional_vars', None):
        try:
            for z in node.optional_vars:
                vars_list.append(self.visit(z))
        except TypeError:  # Not iterable.
            vars_list.append(self.visit(node.optional_vars))
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            result.append(self.visit(item.context_expr))
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        vars_list.append(self.visit(z))
                except TypeError:  # Not iterable.
                    vars_list.append(self.visit(item.optional_vars))
    result.append(','.join(vars_list))
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append('\n')
    return ''.join(result)

def do_AsyncWith(self, node):
    return self.do_With(node, async_flag=True)
#@+node:ekr.20141012064706.18466: *6* f.Yield
def do_Yield(self, node):
    if getattr(node, 'value', None):
        return self.indent(f'yield %s\n' % (
            self.visit(node.value)))
    return self.indent('yield\n')
#@+node:ekr.20160317050557.5: *6* f.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    return self.indent(f'yield from %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18471: *4* class AstFullTraverser
class AstFullTraverser:
    """
    A fast traverser for AST trees: it visits every node (except node.ctx fields).

    Sets .context and .parent ivars before visiting each node.
    """

    def __init__(self):
        """Ctor for AstFullTraverser class."""
        self.context = None
        self.level = 0  # The context level only.
        self.parent = None

    @others
#@+node:ekr.20141012064706.18472: *5* ft.contexts
#@+node:ekr.20141012064706.18473: *6* ft.ClassDef
# 2: ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, visit_body=True):
    old_context = self.context
    self.context = node
    self.level += 1
    for z in node.decorator_list:
        self.visit(z)
    for z in node.bases:
        self.visit(z)
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            self.visit(keyword.value)
    if getattr(node, 'starargs', None):  # Python 3
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):  # Python 3
        self.visit(node.kwargs)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context
#@+node:ekr.20141012064706.18474: *6* ft.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, visit_body=True):

    old_context = self.context
    self.context = node
    self.level += 1
    # Visit the tree in token order.
    for z in node.decorator_list:
        self.visit(z)
    assert isinstance(node.name, str)
    self.visit(node.args)
    if getattr(node, 'returns', None):  # Python 3.
        self.visit(node.returns)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context

do_AsyncFunctionDef = do_FunctionDef
#@+node:ekr.20141012064706.18475: *6* ft.Interactive
def do_Interactive(self, node):
    assert False, 'Interactive context not supported'
#@+node:ekr.20141012064706.18476: *6* ft.Lambda
# Lambda(arguments args, expr body)

def do_Lambda(self, node):
    old_context = self.context
    self.context = node
    self.visit(node.args)
    self.visit(node.body)
    self.context = old_context
#@+node:ekr.20141012064706.18477: *6* ft.Module
def do_Module(self, node):
    self.context = node
    for z in node.body:
        self.visit(z)
    self.context = None
#@+node:ekr.20141012064706.18478: *5* ft.ctx nodes
# Not used in this class, but may be called by subclasses.

def do_AugLoad(self, node):
    pass

def do_Del(self, node):
    pass

def do_Load(self, node):
    pass

def do_Param(self, node):
    pass

def do_Store(self, node):
    pass
#@+node:ekr.20171214200319.1: *5* ft.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    s = AstFormatter().format(node, level, *args, **keys)
    return s.rstrip()
#@+node:ekr.20141012064706.18480: *5* ft.operators & operands
#@+node:ekr.20160521102250.1: *6* ft.op_name
def op_name(self, node, strict=True):
    """Return the print name of an operator node."""
    name = _op_names.get(node.__class__.__name__, f'<%s>' % node.__class__.__name__)
    if strict:
        assert name, node.__class__.__name__
    return name
#@+node:ekr.20141012064706.18482: *6* ft.arguments & arg
# 2: arguments = (
# expr* args,
#   identifier? vararg,
#   identifier? kwarg,
#   expr* defaults)
# 3: arguments = (
#   arg*  args,
#   arg? vararg,
#   arg* kwonlyargs,
#   expr* kw_defaults,
#   arg? kwarg,
#   expr* defaults)

def do_arguments(self, node):

    for z in node.args:
        self.visit(z)
    if getattr(node, 'vararg', None):
        # An identifier in Python 2.
        self.visit(node.vararg)
    if getattr(node, 'kwarg', None):
        # An identifier in Python 2.
        self.visit_list(node.kwarg)
    if getattr(node, 'kwonlyargs', None):  # Python 3.
        self.visit_list(node.kwonlyargs)
    if getattr(node, 'kw_defaults', None):  # Python 3.
        self.visit_list(node.kw_defaults)
    for z in node.defaults:
        self.visit(z)

# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        self.visit(node.annotation)
#@+node:ekr.20141012064706.18483: *6* ft.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    self.visit(node.value)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18484: *6* ft.BinOp
# BinOp(expr left, operator op, expr right)

def do_BinOp(self, node):
    self.visit(node.left)
    # self.op_name(node.op)
    self.visit(node.right)
#@+node:ekr.20141012064706.18485: *6* ft.BoolOp
# BoolOp(boolop op, expr* values)

def do_BoolOp(self, node):
    for z in node.values:
        self.visit(z)
#@+node:ekr.20141012064706.18481: *6* ft.Bytes
def do_Bytes(self, node):
    pass  # Python 3.x only.
#@+node:ekr.20141012064706.18486: *6* ft.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):
    # Call the nodes in token order.
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
#@+node:ekr.20141012064706.18487: *6* ft.Compare
# Compare(expr left, cmpop* ops, expr* comparators)

def do_Compare(self, node):
    # Visit all nodes in token order.
    self.visit(node.left)
    assert len(node.ops) == len(node.comparators)
    for i in range(len(node.ops)):
        self.visit(node.ops[i])
        self.visit(node.comparators[i])
    # self.visit(node.left)
    # for z in node.comparators:
        # self.visit(z)
#@+node:ekr.20150526140323.1: *6* ft.Compare ops
# Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn

def do_Eq(self, node): pass

def do_Gt(self, node): pass

def do_GtE(self, node): pass

def do_In(self, node): pass

def do_Is(self, node): pass

def do_IsNot(self, node): pass

def do_Lt(self, node): pass

def do_LtE(self, node): pass

def do_NotEq(self, node): pass

def do_NotIn(self, node): pass
#@+node:ekr.20141012064706.18488: *6* ft.comprehension
# comprehension (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):
    self.visit(node.target)  # A name.
    self.visit(node.iter)  # An attribute.
    for z in node.ifs:
        self.visit(z)
#@+node:ekr.20170721073315.1: *6* ft.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    pass
#@+node:ekr.20141012064706.18489: *6* ft.Dict
# Dict(expr* keys, expr* values)

def do_Dict(self, node):
    # Visit all nodes in token order.
    assert len(node.keys) == len(node.values)
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.visit(node.values[i])
#@+node:ekr.20160523094910.1: *6* ft.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    # EKR: visit generators first, then value.
    for z in node.generators:
        self.visit(z)
    self.visit(node.value)
    self.visit(node.key)
#@+node:ekr.20150522081707.1: *6* ft.Ellipsis
def do_Ellipsis(self, node):
    pass
#@+node:ekr.20141012064706.18490: *6* ft.Expr
# Expr(expr value)

def do_Expr(self, node):
    self.visit(node.value)
#@+node:ekr.20141012064706.18491: *6* ft.Expression
def do_Expression(self, node):
    """An inner expression"""
    self.visit(node.body)
#@+node:ekr.20141012064706.18492: *6* ft.ExtSlice
def do_ExtSlice(self, node):
    for z in node.dims:
        self.visit(z)
#@+node:ekr.20170721075714.1: *6* ft.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
#@+node:ekr.20141012064706.18493: *6* ft.GeneratorExp
# GeneratorExp(expr elt, comprehension* generators)

def do_GeneratorExp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18494: *6* ft.ifExp (ternary operator)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):
    self.visit(node.body)
    self.visit(node.test)
    self.visit(node.orelse)
#@+node:ekr.20141012064706.18495: *6* ft.Index
def do_Index(self, node):
    self.visit(node.value)
#@+node:ekr.20170721080935.1: *6* ft.JoinedStr (Python 3.6+)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20141012064706.18496: *6* ft.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    self.visit(node.value)
#@+node:ekr.20141012064706.18497: *6* ft.List & ListComp
# List(expr* elts, expr_context ctx)

def do_List(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18498: *6* ft.Name (revise)
# Name(identifier id, expr_context ctx)

def do_Name(self, node):
    # self.visit(node.ctx)
    pass

def do_NameConstant(self, node):  # Python 3 only.
    pass
    # s = repr(node.value)
    # return 'bool' if s in ('True', 'False') else s
#@+node:ekr.20150522081736.1: *6* ft.Num
def do_Num(self, node):
    pass  # Num(object n) # a number as a PyObject.
#@+node:ekr.20141012064706.18499: *6* ft.Repr
# Python 2.x only
# Repr(expr value)

def do_Repr(self, node):
    self.visit(node.value)
#@+node:ekr.20160523094939.1: *6* ft.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)

#@+node:ekr.20160523095142.1: *6* ft.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):
    # EKR: visit generators first.
    for z in node.generators:
        self.visit(z)
    self.visit(node.elt)
#@+node:ekr.20141012064706.18500: *6* ft.Slice
def do_Slice(self, node):
    if getattr(node, 'lower', None):
        self.visit(node.lower)
    if getattr(node, 'upper', None):
        self.visit(node.upper)
    if getattr(node, 'step', None):
        self.visit(node.step)
#@+node:ekr.20150522081748.1: *6* ft.Str
def do_Str(self, node):
    pass  # represents a string constant.
#@+node:ekr.20141012064706.18501: *6* ft.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    self.visit(node.value)
    self.visit(node.slice)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18502: *6* ft.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18503: *6* ft.UnaryOp
# UnaryOp(unaryop op, expr operand)

def do_UnaryOp(self, node):
    # self.op_name(node.op)
    self.visit(node.operand)
#@+node:ekr.20141012064706.18504: *5* ft.statements
#@+node:ekr.20141012064706.18505: *6* ft.alias
# identifier name, identifier? asname)

def do_alias(self, node):
    # self.visit(node.name)
    # if getattr(node,'asname')
        # self.visit(node.asname)
    pass
#@+node:ekr.20170721074528.1: *6* ft.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    self.visit(node.target)
    self.visit(node.annotation)
    self.visit(node.value)
#@+node:ekr.20141012064706.18506: *6* ft.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):
    self.visit(node.test)
    if node.msg:
        self.visit(node.msg)
#@+node:ekr.20141012064706.18507: *6* ft.Assign
# Assign(expr* targets, expr value)

def do_Assign(self, node):
    for z in node.targets:
        self.visit(z)
    self.visit(node.value)
#@+node:ekr.20141012064706.18508: *6* ft.AugAssign
# AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    self.visit(node.target)
    self.visit(node.value)
#@+node:ekr.20141012064706.18509: *6* ft.Break
def do_Break(self, tree):
    pass
#@+node:ekr.20141012064706.18510: *6* ft.Continue
def do_Continue(self, tree):
    pass
#@+node:ekr.20141012064706.18511: *6* ft.Delete
# Delete(expr* targets)

def do_Delete(self, node):
    for z in node.targets:
        self.visit(z)
#@+node:ekr.20141012064706.18512: *6* ft.ExceptHandler
# Python 2: ExceptHandler(expr? type, expr? name, stmt* body)
# Python 3: ExceptHandler(expr? type, identifier? name, stmt* body)

def do_ExceptHandler(self, node):

    if node.type:
        self.visit(node.type)
    if node.name and isinstance(node.name, ast.Name):
        self.visit(node.name)
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18513: *6* ft.Exec
# Python 2.x only
# Exec(expr body, expr? globals, expr? locals)

def do_Exec(self, node):
    self.visit(node.body)
    if getattr(node, 'globals', None):
        self.visit(node.globals)
    if getattr(node, 'locals', None):
        self.visit(node.locals)
#@+node:ekr.20141012064706.18514: *6* ft.For & AsyncFor
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node):
    self.visit(node.target)
    self.visit(node.iter)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)

do_AsyncFor = do_For
#@+node:ekr.20141012064706.18515: *6* ft.Global
# Global(identifier* names)

def do_Global(self, node):
    pass
#@+node:ekr.20141012064706.18516: *6* ft.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node):
    self.visit(node.test)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18517: *6* ft.Import & ImportFrom
# Import(alias* names)

def do_Import(self, node):
    pass
# ImportFrom(identifier? module, alias* names, int? level)

def do_ImportFrom(self, node):
    # for z in node.names:
        # self.visit(z)
    pass
#@+node:ekr.20160317051434.2: *6* ft.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    pass
#@+node:ekr.20141012064706.18518: *6* ft.Pass
def do_Pass(self, node):
    pass
#@+node:ekr.20141012064706.18519: *6* ft.Print
# Python 2.x only
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):
    if getattr(node, 'dest', None):
        self.visit(node.dest)
    for expr in node.values:
        self.visit(expr)
#@+node:ekr.20141012064706.18520: *6* ft.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    for attr in ('exc', 'cause'):
        if getattr(node, attr, None):
            self.visit(getattr(node, attr))
#@+node:ekr.20141012064706.18521: *6* ft.Return
# Return(expr? value)

def do_Return(self, node):
    if node.value:
        self.visit(node.value)
#@+node:ekr.20160317051434.3: *6* ft.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.visit(node.value)
#@+node:ekr.20141012064706.18522: *6* ft.Try (Python 3)
# Python 3 only: Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18523: *6* ft.TryExcept
# TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)

def do_TryExcept(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18524: *6* ft.TryFinally
# TryFinally(stmt* body, stmt* finalbody)

def do_TryFinally(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18525: *6* ft.While
# While(expr test, stmt* body, stmt* orelse)

def do_While(self, node):
    self.visit(node.test)  # Bug fix: 2013/03/23.
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18526: *6* ft.With & AsyncWith
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node):
    if getattr(node, 'context_expr', None):
        self.visit(node.context_expr)
    if getattr(node, 'optional_vars', None):
        self.visit(node.optional_vars)
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        self.visit(z)
                except TypeError:  # Not iterable.
                    self.visit(item.optional_vars)
    for z in node.body:
        self.visit(z)

do_AsyncWith = do_With
#@+node:ekr.20141012064706.18527: *6* ft.Yield, YieldFrom & Await (Python 3)
# Yield(expr? value)
# Await(expr value)         Python 3 only.
# YieldFrom (expr value)    Python 3 only.

def do_Yield(self, node):
    if node.value:
        self.visit(node.value)

do_Await = do_YieldFrom = do_Yield
#@+node:ekr.20141012064706.18528: *5* ft.visit (supports before_* & after_*)
def visit(self, node):
    """Visit a *single* ast node.  Visitors are responsible for visiting children!"""
    name = node.__class__.__name__
    assert isinstance(node, ast.AST), repr(node)
    # Visit the children with the new parent.
    old_parent = self.parent
    self.parent = node
    before_method = getattr(self, 'before_'+name, None)
    if before_method:
        before_method(node)
    do_method = getattr(self, 'do_'+name, None)
    if do_method:
        val = do_method(node)
    after_method = getattr(self, 'after_'+name, None)
    if after_method:
        after_method(node)
    self.parent = old_parent
    return val

def visit_children(self, node):
    assert False, 'must visit children explicitly'
#@+node:ekr.20141012064706.18529: *5* ft.visit_list
def visit_list(self, aList):
    """Visit all ast nodes in aList or ast.node."""
    if isinstance(aList, (list, tuple)):
        for z in aList:
            self.visit(z)
        return None
    assert isinstance(aList, ast.AST), repr(aList)
    return self.visit(aList)
#@+node:ekr.20141012064706.18530: *4* class AstPatternFormatter (AstFormatter)
class AstPatternFormatter(AstFormatter):
    """
    A subclass of AstFormatter that replaces values of constants by Bool,
    Bytes, Int, Name, Num or Str.
    """
    # No ctor.
    @others
#@+node:ekr.20141012064706.18531: *5* Constants & Name
# Return generic markers allow better pattern matches.

def do_BoolOp(self, node):  # Python 2.x only.
    return 'Bool'

def do_Bytes(self, node):  # Python 3.x only.
    return 'Bytes'  # return str(node.s)

def do_Constant(self, node):  # Python 3.6+ only.
    return 'Constant'

def do_Name(self, node):
    return 'Bool' if node.id in ('True', 'False') else node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return 'Bool' if s in ('True', 'False') else s

def do_Num(self, node):
    return 'Num'  # return repr(node.n)

def do_Str(self, node):
    """This represents a string constant."""
    return 'Str'  # return repr(node.s)
#@+node:ekr.20150722204300.1: *4* class HTMLReportTraverser
class HTMLReportTraverser:
    """
    Create html reports from an AST tree.

    Inspired by Paul Boddie.

    This version writes all html to a global code list.

    At present, this code does not show comments.
    The TokenSync class is probably the best way to do this.
    """
    # To do: revise report-traverser-debug.css.
    @others
#@+node:ekr.20150722204300.2: *5* rt.__init__
def __init__(self, debug=False):
    """Ctor for the NewHTMLReportTraverser class."""
    self.code_list = []
    self.debug = debug
    self.div_stack = []
        # A check to ensure matching div/end_div.
    self.last_doc = None
    # List of divs & spans to generate...
    self.enable_list = [
        'body', 'class', 'doc', 'function',
        'keyword', 'name', 'statement'
    ]
    # Formatting stuff...
    debug_css = 'report-traverser-debug.css'
    plain_css = 'report-traverser.css'
    self.css_fn = debug_css if debug else plain_css
    self.html_footer = '\n</body>\n</html>\n'
    self.html_header = self.define_html_header()
#@+node:ekr.20150722204300.3: *6* define_html_header
def define_html_header(self):
    # Use string catenation to avoid using g.adjustTripleString.
    return (
        '<?xml version="1.0" encoding="iso-8859-15"?>\n'
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"\n'
        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n'
        '<html xmlns="http://www.w3.org/1999/xhtml">\n'
        '<head>\n'
        '  <title>%(title)s</title>\n'
        '  <link rel="stylesheet" type="text/css" href="%(css-fn)s" />\n'
        '</head>\n<body>'
    )
#@+node:ekr.20150723094359.1: *5* rt.code generators
#@+node:ekr.20150723100236.1: *6* rt.blank
def blank(self):
    """Insert a single blank."""
    self.clean(' ')
    if self.code_list[-1] not in ' \n':
        self.gen(' ')
#@+node:ekr.20150723100208.1: *6* rt.clean
def clean(self, s):
    """Remove s from the code list."""
    s2 = self.code_list[-1]
    if s2 == s:
        self.code_list.pop()
#@+node:ekr.20150723105702.1: *6* rt.colon
def colon(self):

    self.clean('\n')
    self.clean(' ')
    self.clean('\n')
    self.gen(':')
#@+node:ekr.20150723100346.1: *6* rt.comma & clean_comma
def comma(self):

    self.clean(' ')
    self.gen(', ')

def clean_comma(self):

    self.clean(', ')
#@+node:ekr.20150722204300.21: *6* rt.doc
# Called by ClassDef & FunctionDef visitors.

def doc(self, node):
    doc = ast.get_docstring(node)
    if doc:
        self.docstring(doc)
        self.last_doc = doc  # Attempt to suppress duplicate.
#@+node:ekr.20150722204300.22: *6* rt.docstring
def docstring(self, s):

    import textwrap
    self.gen("<pre class='doc'>")
    self.gen('"""')
    self.gen(self.text(textwrap.dedent(s.replace('"""', '\\"\\"\\"'))))
    self.gen('"""')
    self.gen("</pre>")
#@+node:ekr.20150722211115.1: *6* rt.gen
def gen(self, s):
    """Append s to the global code list."""
    if s:
        self.code_list.append(s)
#@+node:ekr.20150722204300.23: *6* rt.keyword (code generator)
def keyword(self, name):

    self.blank()
    self.span('keyword')
    self.gen(name)
    self.end_span('keyword')
    self.blank()
#@+node:ekr.20150722204300.24: *6* rt.name
def name(self, name):

    # Div would put each name on a separate line.
    # span messes up whitespace, for now.
    # self.span('name')
    self.gen(name)
    # self.end_span('name')
#@+node:ekr.20150723100417.1: *6* rt.newline
def newline(self):

    self.clean(' ')
    self.clean('\n')
    self.clean(' ')
    self.gen('\n')
#@+node:ekr.20150722204300.26: *6* rt.op
def op(self, op_name, leading=False, trailing=True):

    if leading:
        self.blank()
    # self.span('operation')
    # self.span('operator')
    self.gen(self.text(op_name))
    # self.end_span('operator')
    if trailing:
        self.blank()
    # self.end_span('operation')
#@+node:ekr.20160315184954.1: *6* rt.string (code generator)
def string(self, s):

    import xml.sax.saxutils as saxutils
    s = repr(s.strip().strip())
    s = saxutils.escape(s)
    self.gen(s)
#@+node:ekr.20150722204300.27: *6* rt.simple_statement
def simple_statement(self, name):

    class_name = f'%s nowrap' % name
    self.div(class_name)
    self.keyword(name)
    self.end_div(class_name)
#@+node:ekr.20150722204300.16: *5* rt.html helpers
#@+node:ekr.20150722204300.17: *6* rt.attr & text
def attr(self, s):
    return self.text(s).replace("'", "&apos;").replace('"', "&quot;")

def text(self, s):
    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
#@+node:ekr.20150722204300.18: *6* rt.br
def br(self):
    return '\n<br />'
#@+node:ekr.20150722204300.19: *6* rt.comment
def comment(self, comment):

    self.span('comment')
    self.gen('# '+comment)
    self.end_span('comment')
    self.newline()
#@+node:ekr.20150722204300.20: *6* rt.div
def div(self, class_name, extra=None, wrap=False):
    """Generate the start of a div element."""
    if class_name in self.enable_list:
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
        self.newline()
        if class_name and extra:
            self.gen(f"<div class='%s' %s>" % (full_class_name, extra))
        elif class_name:
            self.newline()
            self.gen(f"<div class='%s'>" % (full_class_name))
        else:
            assert not extra
            self.gen("<div>")
    self.div_stack.append(class_name)
#@+node:ekr.20150722222149.1: *6* rt.div_body
def div_body(self, aList):
    if aList:
        self.div_list('body', aList)
#@+node:ekr.20150722221101.1: *6* rt.div_list & div_node
def div_list(self, class_name, aList, sep=None):

    self.div(class_name)
    self.visit_list(aList, sep=sep)
    self.end_div(class_name)

def div_node(self, class_name, node):

    self.div(class_name)
    self.visit(node)
    self.end_div(class_name)
#@+node:ekr.20150723095033.1: *6* rt.end_div
def end_div(self, class_name):

    if class_name in self.enable_list:
        # self.newline()
        self.gen('</div>')
        # self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150723095004.1: *6* rt.end_span
def end_span(self, class_name):

    if class_name in self.enable_list:
        self.gen('</span>')
        self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150722221408.1: *6* rt.keyword_colon
# def keyword_colon(self, keyword):

    # self.keyword(keyword)
    # self.colon()
#@+node:ekr.20150722204300.5: *6* rt.link
def link(self, class_name, href, a_text):

    return f"<a class='%s' href='%s'>%s</a>" % (
        class_name, href, a_text)
#@+node:ekr.20150722204300.6: *6* rt.module_link
def module_link(self, module_name, classes=None):

    return self.link(
        class_name=classes or 'name',
        href=f'%s.xhtml' % module_name,
        a_text=self.text(module_name))
#@+node:ekr.20150722204300.7: *6* rt.name_link
def name_link(self, module_name, full_name, name, classes=None):

    return self.link(
        class_name=classes or "specific-ref",
        href=f'%s.xhtml#%s' % (module_name, self.attr(full_name)),
        a_text=self.text(name))
#@+node:ekr.20150722204300.8: *6* rt.object_name_ref
def object_name_ref(self, module, obj, name=None, classes=None):
    """
    Link to the definition for 'module' using 'obj' with the optional 'name'
    used as the label (instead of the name of 'obj'). The optional 'classes'
    can be used to customise the CSS classes employed.
    """
    return self.name_link(
        module.full_name(),
        obj.full_name(),
        name or obj.name, classes)
#@+node:ekr.20150722204300.9: *6* rt.popup
def popup(self, classes, aList):

    self.span_list(classes or 'popup', aList)
#@+node:ekr.20150722204300.28: *6* rt.span
def span(self, class_name, wrap=False):

    if class_name in self.enable_list:
        self.newline()
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
            self.gen(f"<span class='%s'>" % (full_class_name))
        else:
            self.gen('<span>')
        # self.newline()
    self.div_stack.append(class_name)
#@+node:ekr.20150722224734.1: *6* rt.span_list & span_node
def span_list(self, class_name, aList, sep=None):

    self.span(class_name)
    self.visit_list(aList, sep=sep)
    self.end_span(class_name)

def span_node(self, class_name, node):

    self.span(class_name)
    self.visit(node)
    self.end_span(class_name)
#@+node:ekr.20150722204300.10: *6* rt.summary_link
def summary_link(self, module_name, full_name, name, classes=None):

    return self.name_link(
        f"{module_name}-summary", full_name, name, classes)
#@+node:ekr.20160315161259.1: *5* rt.main
def main(self, fn, node):
    """Return a report for the given ast node as a string."""
    self.gen(self.html_header % {
            'css-fn': self.css_fn,
            'title': f"Module: {fn}"
        })
    self.parent = None
    self.parents = [None]
    self.visit(node)
    self.gen(self.html_footer)
    return ''.join(self.code_list)
#@+node:ekr.20150722204300.44: *5* rt.visit
def visit(self, node):
    """Walk a tree of AST nodes."""
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    method(node)
#@+node:ekr.20150722204300.45: *5* rt.visit_list
def visit_list(self, aList, sep=None):
    # pylint: disable=arguments-differ
    if aList:
        for z in aList:
            self.visit(z)
            self.gen(sep)
        self.clean(sep)
#@+node:ekr.20150722204300.46: *5* rt.visitors
#@+node:ekr.20170721074613.1: *6* rt.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):

    self.div('statement')
    self.visit(node.target)
    self.op('=:', leading=True, trailing=True)
    self.visit(node.annotation)
    self.blank()
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.49: *6* rt.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):

    self.div('statement')
    self.keyword("assert")
    self.visit(node.test)
    if node.msg:
        self.comma()
        self.visit(node.msg)
    self.end_div('statement')
#@+node:ekr.20150722204300.50: *6* rt.Assign
def do_Assign(self, node):

    self.div('statement')
    for z in node.targets:
        self.visit(z)
        self.op('=', leading=True, trailing=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.51: *6* rt.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):

    self.visit(node.value)
    self.gen('.')
    self.gen(node.attr)
#@+node:ekr.20160523102939.1: *6* rt.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    self.div('statement')
    self.keyword('await')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.52: *6* rt.AugAssign
#  AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    op_name_ = op_name(node.op)
    self.div('statement')
    self.visit(node.target)
    self.op(op_name_, leading=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.53: *6* rt.BinOp
def do_BinOp(self, node):

    op_name_ = op_name(node.op)
    # self.span(op_name_)
    self.visit(node.left)
    self.op(op_name_, leading=True)
    self.visit(node.right)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.54: *6* rt.BoolOp
def do_BoolOp(self, node):

    op_name_ = op_name(node.op).strip()
    self.span(op_name_)
    for i, node2 in enumerate(node.values):
        if i > 0:
            self.keyword(op_name_)
        self.visit(node2)
    self.end_span(op_name_)
#@+node:ekr.20150722204300.55: *6* rt.Break
def do_Break(self, node):

    self.simple_statement('break')
#@+node:ekr.20160523103529.1: *6* rt.Bytes (Python 3)
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20150722204300.56: *6* rt.Call & do_keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    # self.span("callfunc")
    self.visit(node.func)
    # self.span("call")
    self.gen('(')
    self.visit_list(node.args, sep=',')
    if node.keywords:
        self.visit_list(node.keywords, sep=',')
    if getattr(node, 'starargs', None):
        self.op('*', trailing=False)
        self.visit(node.starargs)
        self.comma()
    if getattr(node, 'kwargs', None):
        self.op('**', trailing=False)
        self.visit(node.kwargs)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('call')
    # self.end_span('callfunc')
#@+node:ekr.20150722204300.57: *7* rt.do_keyword
# keyword = (identifier arg, expr value)
# keyword arguments supplied to call

def do_keyword(self, node):

    self.span('keyword-arg')
    self.gen(node.arg)
    self.blank()
    self.gen('=')
    self.blank()
    self.visit(node.value)
    self.end_span('keyword-arg')
#@+node:ekr.20150722204300.58: *6* rt.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node):

    has_bases = (node.bases or hasattr(node, 'keywords') or
        hasattr(node, 'starargs') or hasattr(node, 'kwargs'))
    self.div('class')
    self.keyword("class")
    self.gen(node.name)  # Always a string.
    if has_bases:
        self.gen('(')
        self.visit_list(node.bases, sep=', ')
        if getattr(node, 'keywords', None):  # Python 3
            for keyword in node.keywords:
                self.gen(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
        if getattr(node, 'starargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.starargs))
        if getattr(node, 'kwargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.kwargs))
        self.gen(')')
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('class')
#@+node:ekr.20150722204300.59: *6* rt.Compare
def do_Compare(self, node):

    assert len(node.ops) == len(node.comparators)
    # self.span('compare')
    self.visit(node.left)
    for i in range(len(node.ops)):
        op_name_ = op_name(node.ops[i])
        self.op(op_name_, leading=True)
        self.visit(node.comparators[i])
    # self.end_span('compare')
#@+node:ekr.20150722204300.60: *6* rt.comprehension
# comprehension = (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):

    self.visit(node.target)
    self.keyword('in')
    # self.span('collection')
    self.visit(node.iter)
    if node.ifs:
        self.keyword('if')
        # self.span_list("conditional", node.ifs, sep=' ')
        for z in node.ifs:
            self.visit(z)
            self.blank()
        self.clean(' ')
    # self.end_span('collection')
#@+node:ekr.20170721073431.1: *6* rt.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20150722204300.61: *6* rt.Continue
def do_Continue(self, node):

    self.simple_statement('continue')
#@+node:ekr.20150722204300.62: *6* rt.Delete
def do_Delete(self, node):

    self.div('statement')
    self.keyword('del')
    if node.targets:
        self.visit_list(node.targets, sep=',')
    self.end_div('statement')
#@+node:ekr.20150722204300.63: *6* rt.Dict
def do_Dict(self, node):

    assert len(node.keys) == len(node.values)
    # self.span('dict')
    self.gen('{')
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.colon()
        self.visit(node.values[i])
        self.comma()
    self.clean_comma()
    self.gen('}')
    # self.end_span('dict')
#@+node:ekr.20160523104330.1: *6* rt.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.47: *6* rt.do_arguments & helpers
# arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults)

def do_arguments(self, node):

    assert isinstance(node, ast.arguments), node
    first_default = len(node.args) - len(node.defaults)
    for n, arg in enumerate(node.args):
        if isinstance(arg, (list, tuple)):
            self.tuple_parameter(arg)
        else:
            self.visit(arg)
        if n >= first_default:
            default = node.defaults[n - first_default]
            self.gen("=")
            self.visit(default)
        self.comma()
    if getattr(node, 'vararg', None):
        self.gen('*')
        self.gen(self.name(node.vararg))
        self.comma()
    if getattr(node, 'kwarg', None):
        self.gen('**')
        self.gen(self.name(node.kwarg))
        self.comma()
    self.clean_comma()
#@+node:ekr.20160315182225.1: *7* rt.arg (Python 3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):

    self.gen(node.arg)
    if getattr(node, 'annotation', None):
        self.colon()
        self.visit(node.annotation)
#@+node:ekr.20150722204300.48: *7* rt.tuple_parameter
def tuple_parameter(self, node):

    assert isinstance(node, (list, tuple)), node
    self.gen("(")
    for param in node:
        if isinstance(param, tuple):
            self.tuple_parameter(param)
        else:
            self.visit(param)
    self.gen(")")
#@+node:ekr.20150722204300.64: *6* rt.Ellipsis
def do_Ellipsis(self, node):

    self.gen('...')
#@+node:ekr.20150722204300.65: *6* rt.ExceptHandler
def do_ExceptHandler(self, node):

    self.div('excepthandler')
    self.keyword("except")
    if not node.type:
        self.clean(' ')
    if node.type:
        self.visit(node.type)
    if node.name:
        self.keyword('as')
        self.visit(node.name)
    self.colon()
    self.div_body(node.body)
    self.end_div('excepthandler')
#@+node:ekr.20150722204300.66: *6* rt.Exec
# Python 2.x only.

def do_Exec(self, node):

    self.div('statement')
    self.keyword('exec')
    self.visit(node.body)
    if node.globals:
        self.comma()
        self.visit(node.globals)
    if node.locals:
        self.comma()
        self.visit(node.locals)
    self.end_div('statement')
#@+node:ekr.20150722204300.67: *6* rt.Expr
def do_Expr(self, node):

    self.div_node('expr', node.value)
#@+node:ekr.20160523103429.1: *6* rf.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s' % self.visit(node.body)
#@+node:ekr.20160523103751.1: *6* rt.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20150722204300.68: *6* rt.For & AsyncFor (Python 3)
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node, async_flag=False):

    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword("for")
    self.visit(node.target)
    self.keyword("in")
    self.visit(node.iter)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')

def do_AsyncFor(self, node):
    self.do_For(node, async_flag=True)
#@+node:ekr.20170721075845.1: *6* rf.FormattedValue (Python 3.6+: unfinished)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.div('statement')
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
    self.end_div('statement')
#@+node:ekr.20150722204300.69: *6* rt.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False):

    self.div('function', extra=f'id="%s"' % node.name)
    if async_flag:
        self.keyword('async')
    self.keyword("def")
    self.name(node.name)
    self.gen('(')
    self.visit(node.args)
    self.gen(')')
    if getattr(node, 'returns', None):
        self.blank()
        self.gen('->')
        self.blank()
        self.visit(node.returns)
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('function')

def do_AsyncFunctionDef(self, node):
    self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20150722204300.70: *6* rt.GeneratorExp
def do_GeneratorExp(self, node):

    # self.span('genexpr')
    self.gen('(')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span_node('item', node.elt)
    self.visit(node.elt)
    # self.span_list('generators', node.generators)
    self.visit_list(node.generators)
    self.gen(')')
    # self.end_span('genexpr')
#@+node:ekr.20150722204300.71: *6* rt.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert isinstance(ast2, ast.alias), repr(ast2)
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20150722204300.72: *6* rt.Global
def do_Global(self, node):

    self.div('statement')
    self.keyword("global")
    for z in node.names:
        self.gen(z)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20150722204300.73: *6* rt.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node, elif_flag=False):
    
    self.div('statement')
    self.keyword('elif' if elif_flag else 'if')
    self.visit(node.test)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        node1 = node.orelse[0]
        if isinstance(node1, ast.If) and len(node.orelse) == 1:
            self.do_If(node1, elif_flag=True)
        else:
            self.keyword('else')
            self.colon()
            self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.74: *6* rt.IfExp (TernaryOp)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):

    # self.span('ifexp')
    self.visit(node.body)
    self.keyword('if')
    self.visit(node.test)
    self.keyword('else')
    self.visit(node.orelse)
    # self.end_span('ifexp')
#@+node:ekr.20150722204300.75: *6* rt.Import
def do_Import(self, node):

    self.div('statement')
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)  # self.gen(self.module_link(name))
        if alias:
            self.keyword("as")
            self.name(alias)
    self.end_div('statement')
#@+node:ekr.20150722204300.76: *6* rt.ImportFrom
def do_ImportFrom(self, node):

    self.div('statement')
    self.keyword("from")
    self.gen(self.module_link(node.module))
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)
        if alias:
            self.keyword("as")
            self.name(alias)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20160315190818.1: *6* rt.Index
def do_Index(self, node):

    self.visit(node.value)
#@+node:ekr.20170721080959.1: *6* rf.JoinedStr (Python 3.6+: unfinished)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20150722204300.77: *6* rt.Lambda
def do_Lambda(self, node):

    # self.span('lambda')
    self.keyword('lambda')
    self.visit(node.args)
    self.comma()
    self.span_node("code", node.body)
    # self.end_span('lambda')
#@+node:ekr.20150722204300.78: *6* rt.List
# List(expr* elts, expr_context ctx)

def do_List(self, node):

    # self.span('list')
    self.gen('[')
    if node.elts:
        for z in node.elts:
            self.visit(z)
            self.comma()
        self.clean_comma()
    self.gen(']')
    # self.end_span('list')
#@+node:ekr.20150722204300.79: *6* rt.ListComp
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):

    # self.span('listcomp')
    self.gen('[')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span('ifgenerators')
    self.visit_list(node.generators)
    self.gen(']')
    # self.end_span('ifgenerators')
    # self.end_span('listcomp')
#@+node:ekr.20150722204300.80: *6* rt.Module
def do_Module(self, node):

    self.doc(node)
    self.visit_list(node.body)
#@+node:ekr.20150722204300.81: *6* rt.Name
def do_Name(self, node):

    self.name(node.id)
#@+node:ekr.20160315165109.1: *6* rt.NameConstant
def do_NameConstant(self, node):  # Python 3 only.

    self.name(repr(node.value))
#@+node:ekr.20160317051849.2: *6* rt.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    self.div('statement')
    self.keyword('nonlocal')
    self.gen(', '.join(node.names))
    self.end_div('statement')
#@+node:ekr.20150722204300.82: *6* rt.Num
def do_Num(self, node):

    self.gen(self.text(repr(node.n)))
#@+node:ekr.20150722204300.83: *6* rt.Pass
def do_Pass(self, node):

    self.simple_statement('pass')
#@+node:ekr.20150722204300.84: *6* rt.Print
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):

    self.div('statement')
    self.keyword("print")
    self.gen('(')
    if node.dest:
        self.op('>>\n')
        self.visit(node.dest)
        self.comma()
        self.newline()
        if node.values:
            for z in node.values:
                self.visit(z)
                self.comma()
                self.newline()
    self.clean('\n')
    self.clean_comma()
    self.gen(')')
    self.end_div('statement')
#@+node:ekr.20150722204300.85: *6* rt.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    self.div('statement')
    self.keyword("raise")
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            self.visit(getattr(node, attr))
    self.end_div('statement')
#@+node:ekr.20160523105022.1: *6* rt.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20150722204300.86: *6* rt.Return
def do_Return(self, node):

    self.div('statement')
    self.keyword("return")
    if node.value:
        self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160523104433.1: *6* rt.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523104454.1: *6* rt.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.87: *6* rt.Slice
def do_Slice(self, node):

    # self.span("slice")
    if node.lower:
        self.visit(node.lower)
    self.colon()
    if node.upper:
        self.visit(node.upper)
    if node.step:
        self.colon()
        self.visit(node.step)
    # self.end_span("slice")
#@+node:ekr.20160317051849.3: *6* rt.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.gen('*')
    self.visit(node.value)
#@+node:ekr.20150722204300.88: *6* rt.Str
def do_Str(self, node):
    """This represents a string constant."""

    def clean(s):
        return s.replace(' ', '').replace('\n', '').replace('"', '').replace("'", '')

    assert isinstance(node.s, str)
    if self.last_doc and clean(self.last_doc) == clean(node.s):
        # Already seen.
        self.last_doc = None
    else:
        self.string(node.s)
#@+node:ekr.20150722204300.89: *6* rt.Subscript
def do_Subscript(self, node):

    # self.span("subscript")
    self.visit(node.value)
    self.gen('[')
    self.visit(node.slice)
    self.gen(']')
    # self.end_span("subscript")
#@+node:ekr.20160315190913.1: *6* rt.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    if node.finalbody:
        self.keyword('finally')
        self.colon()
        self.div_list('body', node.finalbody)
    self.end_div('statement')
#@+node:ekr.20150722204300.90: *6* rt.TryExcept
def do_TryExcept(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.div_body(node.handlers)
    self.end_div('statement')
#@+node:ekr.20150722204300.91: *6* rt.TryFinally
def do_TryFinally(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_body(node.body)
    self.keyword('finally')
    self.colon()
    self.div_body(node.final.body)
    self.end_div('statement')
#@+node:ekr.20150722204300.92: *6* rt.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):

    # self.span('tuple')
    self.gen('(')
    for z in node.elts or []:
        self.visit(z)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('tuple')
#@+node:ekr.20150722204300.94: *6* rt.While
def do_While(self, node):

    self.div('statement')
    self.div(None)
    self.keyword("while")
    self.visit(node.test)
    self.colon()
    self.end_div(None)
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.93: *6* rt.UnaryOp
def do_UnaryOp(self, node):

    op_name_ = op_name(node.op).strip()
    # self.span(op_name_)
    self.op(op_name_, trailing=False)
    self.visit(node.operand)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.95: *6* rt.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):

    context_expr = getattr(node, 'context_expr', None)
    optional_vars = getattr(node, 'optional_vars', None)
    items = getattr(node, 'items', None)
    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword('with')
    if context_expr:
        self.visit(context_expr)
    if optional_vars:
        self.keyword('as')
        self.visit_list(optional_vars)
    if items:
        for item in items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                self.keyword('as')
                self.visit(item.optional_vars)
    self.colon()
    self.div_body(node.body)
    self.end_div('statement')

def do_AsyncWith(self, node):
    self.do_With(node, async_flag=True)
#@+node:ekr.20150722204300.96: *6* rt.Yield
def do_Yield(self, node):

    self.div('statement')
    self.keyword('yield')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160317051849.5: *6* rt.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    self.div('statement')
    self.keyword('yield from')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160225102931.1: *3* class TokenSync (deprecated)
class TokenSync:
    """A class to sync and remember tokens."""
    # To do: handle comments, line breaks...
    @others
#@+node:ekr.20160225102931.2: *4*  ts.ctor & helpers
def __init__(self, s, tokens):
    """Ctor for TokenSync class."""
    assert isinstance(tokens, list)  # Not a generator.
    self.s = s
    self.first_leading_line = None
    self.lines = [z.rstrip() for z in g.splitLines(s)]
    # Order is important from here on...
    self.nl_token = self.make_nl_token()
    self.line_tokens = self.make_line_tokens(tokens)
    self.blank_lines = self.make_blank_lines()
    self.string_tokens = self.make_string_tokens()
    self.ignored_lines = self.make_ignored_lines()
#@+node:ekr.20160225102931.3: *5* ts.make_blank_lines
def make_blank_lines(self):
    """Return of list of line numbers of blank lines."""
    result = []
    for i, aList in enumerate(self.line_tokens):
        # if any([self.token_kind(z) == 'nl' for z in aList]):
        if len(aList) == 1 and self.token_kind(aList[0]) == 'nl':
            result.append(i)
    return result
#@+node:ekr.20160225102931.4: *5* ts.make_ignored_lines
def make_ignored_lines(self):
    """
    Return a copy of line_tokens containing ignored lines,
    that is, full-line comments or blank lines.
    These are the lines returned by leading_lines().
    """
    result = []
    for i, aList in enumerate(self.line_tokens):
        for z in aList:
            if self.is_line_comment(z):
                result.append(z)
                break
        else:
            if i in self.blank_lines:
                result.append(self.nl_token)
            else:
                result.append(None)
    assert len(result) == len(self.line_tokens)
    for i, aList in enumerate(result):
        if aList:
            self.first_leading_line = i
            break
    else:
        self.first_leading_line = len(result)
    return result
#@+node:ekr.20160225102931.5: *5* ts.make_line_tokens (trace tokens)
def make_line_tokens(self, tokens):
    """
    Return a list of lists of tokens for each list in self.lines.
    The strings in self.lines may end in a backslash, so care is needed.
    """
    import token as tm
    n, result = len(self.lines), []
    for i in range(0, n+1):
        result.append([])
    for token in tokens:
        t1, t2, t3, t4, t5 = token
        kind = tm.tok_name[t1].lower()
        srow, scol = t3
        erow, ecol = t4
        line = erow - 1 if kind == 'string' else srow - 1
        result[line].append(token)
    assert len(self.lines) + 1 == len(result), len(result)
    return result
#@+node:ekr.20160225102931.6: *5* ts.make_nl_token
def make_nl_token(self):
    """Return a newline token with '\n' as both val and raw_val."""
    import token as tm
    t1 = tm.NEWLINE
    t2 = '\n'
    t3 = (0, 0)  # Not used.
    t4 = (0, 0)  # Not used.
    t5 = '\n'
    return t1, t2, t3, t4, t5
#@+node:ekr.20160225102931.7: *5* ts.make_string_tokens
def make_string_tokens(self):
    """Return a copy of line_tokens containing only string tokens."""
    result = []
    for aList in self.line_tokens:
        result.append([z for z in aList if self.token_kind(z) == 'string'])
    assert len(result) == len(self.line_tokens)
    return result
#@+node:ekr.20160225102931.8: *4* ts.check_strings
def check_strings(self):
    """Check that all strings have been consumed."""
    for i, aList in enumerate(self.string_tokens):
        if aList:
            g.trace(f"warning: line {i}. unused strings: {aList}")
#@+node:ekr.20160225102931.10: *4* ts.is_line_comment
def is_line_comment(self, token):
    """Return True if the token represents a full-line comment."""
    import token as tm
    t1, t2, t3, t4, t5 = token
    kind = tm.tok_name[t1].lower()
    raw_val = t5
    return kind == 'comment' and raw_val.lstrip().startswith('#')
#@+node:ekr.20160225102931.12: *4* ts.last_node
def last_node(self, node):
    """Return the node of node's tree with the largest lineno field."""

    class LineWalker(ast.NodeVisitor):

        def __init__(self):
            """Ctor for LineWalker class."""
            self.node = None
            self.lineno = -1

        def visit(self, node):
            """LineWalker.visit."""
            if hasattr(node, 'lineno'):
                if node.lineno > self.lineno:
                    self.lineno = node.lineno
                    self.node = node
            if isinstance(node, list):
                for z in node:
                    self.visit(z)
            else:
                self.generic_visit(node)

    w = LineWalker()
    w.visit(node)
    return w.node
#@+node:ekr.20160225102931.13: *4* ts.leading_lines
def leading_lines(self, node):
    """Return a list of the preceding comment and blank lines"""
    # This can be called on arbitrary nodes.
    leading = []
    if hasattr(node, 'lineno'):
        i, n = self.first_leading_line, node.lineno
        while i < n:
            token = self.ignored_lines[i]
            if token:
                s = self.token_raw_val(token).rstrip() + '\n'
                leading.append(s)
            i += 1
        self.first_leading_line = i
    return leading
#@+node:ekr.20160225102931.14: *4* ts.leading_string
def leading_string(self, node):
    """Return a string containing all lines preceding node."""
    return ''.join(self.leading_lines(node))
#@+node:ekr.20160225102931.15: *4* ts.line_at
def line_at(self, node, continued_lines=True):
    """Return the lines at the node, possibly including continuation lines."""
    n = getattr(node, 'lineno', None)
    if n is None:
        return f'<no line> for %s' % node.__class__.__name__
    if continued_lines:
        aList, n = [], n - 1
        while n < len(self.lines):
            s = self.lines[n]
            if s.endswith('\\'):
                aList.append(s[:-1])
                n += 1
            else:
                aList.append(s)
                break
        return ''.join(aList)
    return self.lines[n - 1]
#@+node:ekr.20160225102931.16: *4* ts.sync_string
def sync_string(self, node):
    """Return the spelling of the string at the given node."""
    n = node.lineno
    tokens = self.string_tokens[n - 1]
    if tokens:
        token = tokens.pop(0)
        self.string_tokens[n - 1] = tokens
        return self.token_val(token)
    g.trace('===== underflow', n, node.s)
    return node.s
#@+node:ekr.20160225102931.18: *4* ts.tokens_for_statement
def tokens_for_statement(self, node):
    assert isinstance(node, ast.AST), node
    name = node.__class__.__name__
    if hasattr(node, 'lineno'):
        tokens = self.line_tokens[node.lineno - 1]
        g.trace(' '.join([self.dump_token(z) for z in tokens]))
    else:
        g.trace('no lineno', name)
#@+node:ekr.20160225102931.19: *4* ts.trailing_comment
def trailing_comment(self, node):
    """
    Return a string containing the trailing comment for the node, if any.
    The string always ends with a newline.
    """
    if hasattr(node, 'lineno'):
        return self.trailing_comment_at_lineno(node.lineno)
    g.trace('no lineno', node.__class__.__name__, g.callers())
    return '\n'
#@+node:ekr.20160225102931.20: *4* ts.trailing_comment_at_lineno
def trailing_comment_at_lineno(self, lineno):
    """Return any trailing comment at the given node.lineno."""
    tokens = self.line_tokens[lineno - 1]
    for token in tokens:
        if self.token_kind(token) == 'comment':
            raw_val = self.token_raw_val(token).rstrip()
            if not raw_val.strip().startswith('#'):
                val = self.token_val(token).rstrip()
                s = f' %s\n' % val
                return s
    return '\n'
#@+node:ekr.20160225102931.21: *4* ts.trailing_lines
def trailing_lines(self):
    """return any remaining ignored lines."""
    trailing = []
    i = self.first_leading_line
    while i < len(self.ignored_lines):
        token = self.ignored_lines[i]
        if token:
            s = self.token_raw_val(token).rstrip() + '\n'
            trailing.append(s)
        i += 1
    self.first_leading_line = i
    return trailing
#@+node:ekr.20191122105543.1: *4* ts:dumps
#@+node:ekr.20160225102931.9: *5* ts.dump_token
def dump_token(self, token, verbose=False):
    """Dump the token. It is either a string or a 5-tuple."""
    import token as tm
    if isinstance(token, str):
        return token
    t1, t2, t3, t4, t5 = token
    kind = g.toUnicode(tm.tok_name[t1].lower())
    # raw_val = g.toUnicode(t5)
    val = g.toUnicode(t2)
    if verbose:
        return f'token: %10s %r' % (kind, val)
    return val
#@+node:ekr.20160225102931.17: *5* ts.token_kind/raw_val/val
def token_kind(self, token):
    """Return the token's type."""
    t1, t2, t3, t4, t5 = token
    import token as tm
    return g.toUnicode(tm.tok_name[t1].lower())

def token_raw_val(self, token):
    """Return the value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t5)

def token_val(self, token):
    """Return the raw value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t2)
#@+node:ekr.20160225102931.11: *5* ts.join
def join(self, aList, sep=','):
    """return the items of the list joined by sep string."""
    tokens = []
    for i, token in enumerate(aList or []):
        tokens.append(token)
        if i < len(aList) - 1:
            tokens.append(sep)
    return tokens
#@+node:ekr.20190910081550.1: *3* class SyntaxSanitizer
class SyntaxSanitizer:

    << SyntaxSanitizer docstring >>

    def __init__(self, c, keep_comments):
        self.c = c
        self.keep_comments = keep_comments

    @others
#@+node:ekr.20190910093739.1: *4* << SyntaxSanitizer docstring >>
r"""
This class converts section references, @others and Leo directives to
comments. This allows ast.parse to handle the result.

Within section references, these comments must *usually* be executable:
    
BEFORE:
    if condition:
        <\< do something >\>
AFTER:
    if condition:
        pass # do something
        
Alas, sanitation can result in a syntax error. For example, leoTips.py contains:
    
BEFORE:
    tips = [
        <\< define tips >\>
        ]

AFTER:
    tips = [
        pass # define tips
    ]
    
This fails because tips = [pass] is a SyntaxError.

The beautify* and black* commands clearly report such failures.
"""
#@+node:ekr.20190910022637.2: *4* sanitize.comment_leo_lines
def comment_leo_lines(self, p=None, s0=None):
    """
    Replace lines containing Leonine syntax with **special comment lines** of the form:
        
        {lws}#{lws}{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    This convention allows uncomment_special_lines to restore these lines.
    """
    # Choose a marker that appears nowhere in s.
    if p:
        s0 = p.b
    n = 5
    while('#'+ ('!'*n)) in s0:
        n += 1
    comment = '#' + ('!' * n)
    # Create a dict of directives.
    d = {z: True for z in g.globalDirectiveList}
    # Convert all Leonine lines to special comments.
    i, lines, result = 0, g.splitLines(s0), []
    while i < len(lines):
        progress = i
        s = lines[i]
        s_lstrip = s.lstrip()
        # Comment out any containing a section reference.
        j = s.find('<<')
        k = s.find('>>') if j > -1 else -1
        if -1 < j < k:
            result.append(comment+s)
            # Generate a properly-indented pass line.
            j2 = g.skip_ws(s, 0)
            result.append(f'{" "*j2}pass\n')
        elif s_lstrip.startswith('@'):
            # Comment out all other Leonine constructs.
            if self.starts_doc_part(s):
                # Comment the entire doc part, until @c or @code.
                result.append(comment+s)
                i += 1
                while i < len(lines):
                    s = lines[i]
                    result.append(comment+s)
                    i += 1
                    if self.ends_doc_part(s):
                        break
            else:
                j = g.skip_ws(s, 0)
                assert s[j] == '@'
                j += 1
                k = g.skip_id(s, j, chars='-')
                if k > j:
                    word = s[j : k]
                    if word == 'others':
                        # Remember the original @others line.
                        result.append(comment+s)
                        # Generate a properly-indented pass line.
                        result.append(f'{" "*(j-1)}pass\n')
                    else:
                        # Comment only Leo directives, not decorators.
                        result.append(comment+s if word in d else s)
                else:
                    result.append(s)
        elif s_lstrip.startswith('#') and self.keep_comments:
            # A leading comment.
            # Bug fix: Preserve lws in comments, too.
            j2 = g.skip_ws(s, 0)
            result.append(" "*j2+comment+s)
        else:
            # A plain line.
            result.append(s)
        if i == progress:
            i += 1
    return comment, ''.join(result)
#@+node:ekr.20190910022637.3: *4* sanitize.starts_doc_part & ends_doc_part
def starts_doc_part(self, s):
    """Return True if s word matches @ or @doc."""
    return s.startswith(('@\n', '@doc\n', '@ ', '@doc '))

def ends_doc_part(self, s):
    """Return True if s word matches @c or @code."""
    return s.startswith(('@c\n', '@code\n', '@c ', '@code '))
#@+node:ekr.20190910022637.4: *4* sanitize.uncomment_leo_lines
def uncomment_leo_lines(self, comment, p, s0):
    """Reverse the effect of comment_leo_lines."""
    lines = g.splitLines(s0)
    i, result = 0, []
    while i < len(lines):
        progress = i
        s = lines[i]
        i += 1
        if comment in s:
            # One or more special lines.
            i = self.uncomment_special_lines(comment, i, lines, p, result, s)
        else:
            # A regular line.
            result.append(s)
        assert progress < i
    return ''.join(result).rstrip() + '\n'
#@+node:ekr.20190910022637.5: *4* sanitize.uncomment_special_line & helpers
def uncomment_special_lines(self, comment, i, lines, p, result, s):
    """
    This method restores original lines from the special comment lines
    created by comment_leo_lines. These lines have the form:
        
        {lws}#{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    s is a line containing the comment delim.
    i points at the *next* line.
    Handle one or more lines, appending stripped lines to result.
    """
    #
    # Delete the lws before the comment.
    # This works because the tail contains the original whitespace.
    assert comment in s
    s = s.lstrip().replace(comment, '')
    #
    # Here, s is the original line.
    if comment in s:
        g.trace(f"can not happen: {s!r}")
        return i
    if self.starts_doc_part(s):
        result.append(s)
        while i < len(lines):
            s = lines[i].lstrip().replace(comment, '')
            i += 1
            result.append(s)
            if self.ends_doc_part(s):
                break
        return i
    j = s.find('<<')
    k = s.find('>>') if j > -1 else -1
    if -1 < j < k or '@others' in s:
        #
        # A section reference line or an @others line.
        # Such lines are followed by a pass line.
        #
        # The beautifier may insert blank lines before the pass line.
        kind = 'section ref' if -1 < j < k else '@others'
        # Restore the original line, including leading whitespace.
        result.append(s)
        # Skip blank lines.
        while i < len(lines) and not lines[i].strip():
            i += 1
        # Skip the pass line.
        if i < len(lines) and lines[i].lstrip().startswith('pass'):
            i += 1
        else:
            g.trace(f"*** no pass after {kind}: {p.h}")
    else:
        # A directive line or a comment line.
        result.append(s)
    return i
#@+node:ekr.20200726125841.1: *3* fs.scan_for_values (token based, not used)
def scan_for_values(self):  # pragma: no cover
    """
    **Important**: This method is not used. It shows how to "parse"
    the RHS of an % operator using tokens instead of a parse tree. 
    
    This is a recursive descent parser. It is comprable in complexity
    to fs.scan_format_string.
    
    Return a list of possibly parenthesized values for the format string.
    
    This method never actually consumes tokens.
    
    If all goes well, we'll skip all tokens in the tokens list.
    """

    # pylint: disable=no-member # This is example code.
    # Skip the '%'
    new_token = self.new_token
    assert self.look_ahead(0) == ('op', '%')
    token_i, tokens = self.skip_ahead(0, 'op', '%')
    # Skip '(' if it's next
    include_paren = self.look_ahead(token_i) == ('op', '(')
    if include_paren:
        token_i, skipped_tokens = self.skip_ahead(token_i, 'op', '(')
        tokens.extend(skipped_tokens)
    # Find all tokens up to the first ')' or 'for'
    values, value_list = [], []
    while token_i < len(self.tokens):
        # Don't use look_ahead here: handle each token exactly once.
        token = self.tokens[token_i]
        token_i += 1
        tokens.append(token)
        kind, val = token.kind, token.value
        if kind == 'ws':
            continue
        if kind in ('newline', 'nl'):
            if include_paren or val.endswith('\\\n'):
                # Continue scanning, ignoring the newline.
                continue
            # The newline ends the scan.
            values.append(value_list)
                # Retain the tokens!
            if not include_paren:  # Bug fix.
                tokens.pop()  # Rescan the ')'
            break
        if (kind, val) == ('op', ')'):
            values.append(value_list)
            if not include_paren:
                tokens.pop()  # Rescan the ')'
            break
        if (kind, val) == ('name', 'for'):
            self.add_trailing_ws = True
            tokens.pop()  # Rescan the 'for'
            values.append(value_list)
            break
        if (kind, val) == ('op', ','):
            values.append(value_list)
            value_list = []
        elif kind == 'op' and val in '([{':
            values_list2, token_i2 = self.scan_to_matching(token_i - 1, val)
            value_list.extend(values_list2)
            tokens.extend(self.tokens[token_i:token_i2])
            token_i = token_i2
        elif kind == 'name':
            # Ensure separation of names.
            value_list.append(new_token(kind, val))
            value_list.append(new_token('ws', ' '))
        else:
            value_list.append(new_token(kind, val))
    return values, tokens
#@+node:ekr.20150605175037.1: ** From leoCheck.py & checkerCommands.py
@first # -*- coding: utf-8 -*-
"""Experimental code checking for Leo."""
# To do:
# - Option to ignore defs without args if all calls have no args.
# * explain typical entries
import importlib
import leo.core.leoGlobals as g
import leo.core.leoAst as leoAst
importlib.reload(leoAst)
import ast
# import glob
import importlib
import os
import re
import time
@others
@language python
@tabwidth -4
@pagewidth 70
#@+node:ekr.20171207095816.1: *3* class ConventionChecker
class ConventionChecker:
    """
    A prototype of an extensible convention-checking tool.
    See: https://github.com/leo-editor/leo-editor/issues/632
    
    Here is the body of @button check-conventions:
    
        g.cls()
        if c.changed: c.save()
        
        import importlib
        import leo.core.leoCheck as leoCheck
        importlib.reload(leoCheck)
        
        fn = g.os_path_finalize_join(g.app.loadDir, '..', 'plugins', 'nodetags.py')
        leoCheck.ConventionChecker(c).check(fn=fn)
    """
    # pylint: disable=literal-comparison
        # What's wrong with `if self.test_kind is 'test'`?

    ignore = ('bool', 'dict', 'enumerate', 'list', 'tuple')
        # Things that look like function calls.

    @others
#@+node:ekr.20171210134449.1: *4* checker.Birth
def __init__(self, c):
    self.c = c
    self.class_name = None
    self.context_stack = []
        # Stack of ClassDef and FunctionDef nodes.
    # Rudimentary symbol tables...
    self.classes = self.init_classes()
    self.special_class_names = [
        'Commands', 'LeoGlobals', 'Position', 'String', 'VNode', 'VNodeBase',
    ]
    self.special_names_dict = self.init_special_names()
    # Debugging
    self.enable_trace = True
    self.file_name = None
    self.indent = 0 # For self.format.
    self.max_time = 0.0
    self.recursion_count = 0
    self.slowest_file = None
    self.stats = self.CCStats()
    # Other ivars...
    self.errors = 0
    self.line_number = 0
    self.pass_n = 0
    self.test_kind = None
    self.unknowns = {} # Keys are expression, values are (line, fn) pairs.
#@+node:ekr.20171209044610.1: *5* checker.init_classes
def init_classes(self):
    """
    Init the symbol tables with known classes.
    """
    return {
        # Pre-enter known classes.
        'Commands': {
            'ivars': {
                'p': self.Type('instance', 'Position'),
            },
            'methods': {},
        },
        'LeoGlobals': {
            'ivars': {}, # g.app, g.app.gui.
            'methods': {
                'trace': self.Type('instance', 'None')
            },
        },
        'Position': {
            'ivars': {
                'v': self.Type('instance', 'VNode'),
                'h': self.Type('instance', 'String'),
            },
            'methods': {},
        },
        'VNode': {
            'ivars': {
                'h': self.Type('instance', 'String'),
                # Vnode has no v instance!
            },
            'methods': {},
        },
        'VNodeBase': {
            'ivars': {},
            'methods': {},
        },
        'String': {
            'ivars': {},
            'methods': {}, # Possible?
        },
    }
    
#@+node:ekr.20171210133853.1: *5* checker.init_special_names
def init_special_names(self):
    """Init known special names."""
    t = self.Type
    return {
        'c': t('instance', 'Commands'),
        'c.p': t('instance', 'Position'),
        'g': t('instance', 'LeoGlobals'), # module?
        'p': t('instance', 'Position'),
        'v': t('instance', 'VNode'),
    }
#@+node:ekr.20171212015700.1: *4* checker.check & helpers (main entry)
def check(self):
    """
    The main entry point for the convention checker.

    A stand-alone version of the @button node that tested the
    ConventionChecker class.
    
    The check-conventions command in checkerCommands.py saves c and
    reloads the leoCheck module before instantiating this class and
    calling this method.
    """
    g.cls()
    c = self.c
    kind = 'production' # <----- Change only this line.
        # 'production', 'project', 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'
    join = g.os_path_finalize_join
    loadDir = g.app.loadDir
    report_stats = True
    files_table = (
        # join(loadDir, 'leoCommands.py'),
        # join(loadDir, 'leoNodes.py'),
        join(loadDir, '..', 'plugins', 'qt_tree.py'),
    )
    # ===== Don't change anything below here =====
    if kind == 'files':
        for fn in files_table:
            self.check_file(fn=fn, trace_fn=True)
    elif kind == 'production':
        for p in g.findRootsWithPredicate(c, c.p, predicate=None):
            self.check_file(fn=g.fullPath(c, p), test_kind=kind, trace_fn=True)
    elif kind in ('project', 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'):
        project_name = 'leo' if kind == 'project' else kind
        self.check_project(project_name)
    elif kind == 'test':
        self.test()
    else:
        g.trace('unknown kind', repr(kind))
    if report_stats:
        self.stats.report()
#@+node:ekr.20171207100432.1: *5* checker.check_file
def check_file(self, fn=None, s=None, test_kind=None, trace_fn=False):
    """Check the contents of fn or the string s."""
    # Get the source.
    if test_kind:
        self.test_kind = test_kind
    if fn:
        sfn = g.shortFileName(fn)
        if g.os_path_exists(fn):
            s, e = g.readFileIntoString(fn)
            if s:
                s = g.toEncodedString(s, encoding=e)
            else:
                g.trace('empty file:', sfn)
                return
        else:
            g.trace('file not found:', sfn)
            return
    elif s:
        sfn = '<string>'
    else:
        g.trace('no fn or s argument')
        return
    # Check the source
    if trace_fn:
        if fn:
            print(f"===== {sfn}")
        else:
            print('===== <string>\n%s\n----- </string>\n' % s.rstrip())
    t1 = time.process_time()
    node = ast.parse(s, filename='before', mode='exec')
    self.check_helper(fn=sfn, node=node, s=s)
    t2 = time.process_time()
    t_tot = t2-t1
    if t_tot > self.max_time:
        self.max_time = t_tot
        self.slowest_file = self.file_name
#@+node:ekr.20171214150828.1: *5* checker.check_helper
def check_helper(self, fn, node, s):

    cct = self.CCTraverser(controller=self)
    for n in 1, 2:
        if self.test_kind == 'test':
            g.trace('===== PASS', n)
        # Init this pass.
        self.file_name = fn
        self.indent = 0
        self.pass_n = n
        cct.visit(node)
    self.end_file()
#@+node:ekr.20171213013004.1: *5* checker.check_project
def check_project(self, project_name):
    
    trace_fn = True
    trace_skipped = False
    self.test_kind = 'project'
    fails_dict = {
        'coverage': ['cmdline.py',],
        'lib2to3': ['fixer_util.py', 'fix_dict.py', 'patcomp.py', 'refactor.py'],
        'leo': [], # All of Leo's core files pass.
        'pylint': [
            'base.py', 'classes.py', 'format.py',
            'logging.py', 'python3.py', 'stdlib.py', 
            'docparams.py', 'lint.py',
        ],
        'rope': ['objectinfo.py', 'objectdb.py', 'runmod.py',],
    }
    fails = fails_dict.get(project_name, [])
    utils = ProjectUtils()
    files = utils.project_files(project_name, force_all=False)
    if files:
        t1 = time.process_time()
        for fn in files:
            sfn = g.shortFileName(fn)
            if sfn in fails or fn in fails:
                if trace_skipped: print('===== skipping', sfn)
            else:
                self.check_file(fn=fn, trace_fn=trace_fn)
        t2 = time.process_time()
        print('%s files in %4.2f sec. max %4.2f sec in %s' % (
            len(files), (t2-t1), self.max_time, self.slowest_file))
        if self.errors:
            print(f"{self.errors} error{g.plural(self.errors)}")
    else:
        print(f"no files for project: {project_name}")
#@+node:ekr.20171208135642.1: *5* checker.end_file & helper
def end_file(self,trace_classes=False, trace_unknowns=False):
    
    # Do *not* clear self.classes.
    self.unknowns = {}
#@+node:ekr.20171212100005.1: *6* checker.trace_unknowns
def trace_unknowns(self):
    print('----- Unknown ivars...')
    d = self.unknowns
    max_key = max([len(key) for key in d ]) if d else 2
    for key, aList in sorted(d.items()):
        # Remove duplicates that vary only in line number.
        aList2, seen = [], []
        for data in aList:
            line, fn, s = data
            data2 = (key, fn, s)
            if data2 not in seen:
                seen.append(data2)
                aList2.append(data)
        for data in aList2:
            line, fn, s = data
            print('%*s %4s %s: %s' % (
                max_key, key, line, fn, g.truncate(s, 60)))
#@+node:ekr.20171212020013.1: *5* checker.test
tests = [
'''\
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
''', # comma required!
]

def test(self):

    for s in self.tests:
        s = g.adjustTripleString(s, self.c.tab_width)
        self.check_file(s=s, test_kind='test', trace_fn=True)
    if self.errors:
        print(f"{self.errors} error{g.plural(self.errors)}")
#@+node:ekr.20171216063026.1: *4* checker.error, fail, note & log_line
def error(self, node, *args, **kwargs):
    
    self.errors += 1
    print('')
    print('Error: %s' % self.log_line(node, *args, **kwargs))
    print('')
    
def fail(self, node, *args, **kwargs):
    self.stats.inference_fails += 1
    print('')
    print('Inference failure: %s' % self.log_line(node, *args, **kwargs))
    print('')
    
def log_line(self, node=None, *args, **kwargs):
    return 'line: %s file: %s: %s' % (
        getattr(node, 'lineno', '??'),
        self.file_name or '<string>',
        ' '.join([z if isinstance(z, str) else repr(z) for z in args]),
    )
    
def note(self, node, *args, **kwargs):

    print('')
    print('Note: %s' % self.log_line(node, *args, **kwargs))
    print('')
#@+node:ekr.20171215080831.1: *4* checker.dump, format
def dump(self, node, annotate_fields=True, level=0, **kwargs):
    """Dump the node."""
    return leoAst.AstDumper().dump(node, level=level)

def format(self, node, *args, **kwargs):
    """Format the node and possibly its descendants, depending on args."""
    s = leoAst.AstFormatter().format(node, level=self.indent, *args, **kwargs)
    return s.rstrip()
#@+node:ekr.20171208142646.1: *4* checker.resolve & helpers
def resolve(self, node, name, context, trace=False):
    """Resolve name in the given context to a Type."""
    self.stats.resolve += 1
    assert isinstance(name, str), (repr(name), g.callers())
    if context:
        if context.kind in ('error', 'unknown'):
            result = context
        elif name == 'self':
            if context.name:
                result = self.Type('instance', context.name)
            else:
                g.trace('===== NO OBJECT NAME')
                result = self.Type('error', 'no object name')
        elif context.kind in ('class', 'instance'):
            result = self.resolve_ivar(node, name, context)
        else:
            result = self.Type('error', f"unknown kind: {context.kind}")
    else:
        result = self.Type('error', f"unbound name: {name}")
    return result
#@+node:ekr.20171208134737.1: *5* checker.resolve_call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def resolve_call(self, node):
    """Resolve the head of the call's chain to a Type."""
    assert self.pass_n == 2
    self.stats.resolve_call += 1
    chain = self.get_chain(node.func)
    if chain:
        func = chain.pop()
        if isinstance(func, ast.Name):
            func = func.id
        assert isinstance(func, str), repr(func)
    if chain:
        assert isinstance(chain[0], ast.Name), repr(chain[0])
        chain[0] = chain[0].id
        # args = ','.join([self.format(z) for z in node.args])
        self.recursion_count = 0
        if self.class_name:
            context = self.Type('instance', self.class_name)
        else:
            context = self.Type('module', self.file_name)
        result = self.resolve_chain(node, chain, context)
    else:
        result = self.Type('unknown', 'empty chain')
    assert isinstance(result, self.Type), repr(result)
    return result
#@+node:ekr.20171209034244.1: *5* checker.resolve_chain
def resolve_chain(self, node, chain, context, trace=False):
    """Resolve the chain to a Type."""
    self.stats.resolve_chain += 1
    name = '<no name>'
    for obj in chain:
        name = obj.id if isinstance(obj, ast.Name) else obj
        assert isinstance(name, str), (repr(name), g.callers())
        context = self.resolve(node, name, context, trace=trace)
    assert isinstance(context, self.Type), repr(context)
    return context
#@+node:ekr.20171208173323.1: *5* checker.resolve_ivar & helpers
def resolve_ivar(self, node, ivar, context):
    """Resolve context.ivar to a Type."""
    assert self.pass_n == 2, repr(self.pass_n)
    self.stats.resolve_ivar += 1
    class_name = 'Commands' if context.name == 'c' else context.name
    self.recursion_count += 1
    if self.recursion_count > 20:
        self.report_unbounded_recursion(node, class_name, ivar, context)
        return self.Type('error', 'recursion')
    the_class = self.classes.get(class_name)
    if not the_class:
        return self.Type('error', f"no class {ivar}")
    ivars = the_class.get('ivars')
    methods = the_class.get('methods')
    if ivar == 'self':
        return self.Type('instance', class_name)
    if methods.get(ivar):
        return self.Type('func', ivar)
    if ivars.get(ivar):
        val = ivars.get(ivar)
        if isinstance(val, self.Type):
            return val
        # Check for pre-defined special names.
        for special_name, special_obj in self.special_names_dict.items():
            tail = val[len(special_name):]
            if val == special_name:
                return special_obj
            if val.startswith(special_name) and tail.startswith('.'):
                # Resovle the rest of the tail in the found context.
                return self.resolve_chain(node, tail[1:], special_obj)
        # Avoid recursion .
        head = val.split('.')
        if ivar in (val, head[0]):
            return self.Type('unknown', ivar)
        for name2 in head:
            old_context = context
            context = self.resolve(node, name2, context)
            if 0: g.trace('recursive %s: %r --> %r' % (name2, old_context, context))
        if 0: g.trace('END RECURSIVE: %r', context)
        return context
    if ivar in self.special_names_dict:
        val = self.special_names_dict.get(ivar)
        return val
    # Remember the unknown.
    self.remember_unknown_ivar(ivar)
    return self.Type('error', f"no member {ivar}")
#@+node:ekr.20171217102701.1: *6* checker.remember_unknown_ivar
def remember_unknown_ivar(self, ivar):

    d = self.unknowns
    aList = d.get(ivar, [])
    data = (self.line_number, self.file_name)
    aList.append(data)
    # tag:setter (data describing unknown ivar)
    d[ivar] = aList
    # self.error(node, 'No member:', ivar)
    return self.Type('error', 'no member %s' % ivar)
#@+node:ekr.20171217102055.1: *6* checker.report_unbounded_recursion
def report_unbounded_recursion(self, node, class_name, ivar, context):
    
    the_class = self.classes.get(class_name)
    self.error(node, 'UNBOUNDED RECURSION: %r %r\nCallers: %s' % (
        ivar, context, g.callers()))
    if 0:
        g.trace('CLASS DICT: Commands')
        g.printDict(self.classes.get('Commands'))
    if 0:
        g.trace('CLASS DICT', class_name)
        g.printDict(the_class)
#@+node:ekr.20171209065852.1: *5* checker_check_signature & helpers
def check_signature(self, node, func, args, signature):
    
    self.stats.check_signature += 1
    if signature[0] == 'self':
        signature = signature[1:]
    result = 'ok'
    for i, arg in enumerate(args):
        if i < len(signature):
            result = self.check_arg(node, func, args, arg, signature[i])
            if result == 'fail':
                self.fail(node, '\n%s(%s) incompatible with %s(%s)' % (
                    func, ','.join(args),
                    func, ','.join(signature),
                ))
                break
    if result == 'ok':
        self.stats.sig_ok += 1
    elif result == 'fail':
        self.stats.sig_fail += 1
    else:
        assert result == 'unknown'
        self.stats.sig_unknown += 1
#@+node:ekr.20171212034531.1: *6* checker.check_arg (Finish)
def check_arg(self, node, func, args, call_arg, sig_arg):
    """
    Check call_arg and sig_arg with arg (a list).
    
    To do: check keyword args.
    """
    return self.check_arg_helper(node, func, call_arg, sig_arg)

#@+node:ekr.20171212035137.1: *6* checker.check_arg_helper
def check_arg_helper(self, node, func, call_arg, sig_arg):

    special_names_dict = self.special_names_dict
    if call_arg == sig_arg or sig_arg in (None, 'None'):
        # Match anything against a default value of None.
        return 'ok'
    # Resolve the call_arg if possible.
    chain = call_arg.split('.')
    if len(chain) > 1:
        head, tail = chain[0], chain[1:]
        if head in special_names_dict:
            context = special_names_dict.get(head)
            context = self.resolve_chain(node, tail, context)
            if context.kind == 'error':
                # Caller will report the error.
                return 'unknown'
            if sig_arg in special_names_dict:
                sig_class = special_names_dict.get(sig_arg)
                return self.compare_classes(
                    node, call_arg, sig_arg, context, sig_class)
    if sig_arg in special_names_dict and call_arg in special_names_dict:
        sig_class = special_names_dict.get(sig_arg)
        call_class = special_names_dict.get(call_arg)
        return self.compare_classes(
            node, call_arg, sig_arg, call_class, sig_class)
    return 'unknown'
#@+node:ekr.20171212044621.1: *6* checker.compare_classes
def compare_classes(self, node, arg1, arg2, class1, class2):

    if class1 == class2:
        self.stats.sig_infer_ok += 1
        return 'ok'
    # The caller reports the failure.
    # self.error(node, 'FAIL', arg1, arg2, class1, class2)
    self.stats.sig_infer_fail += 1
    return 'fail'
#@+node:ekr.20171215074959.1: *4* checker.Visitors & helpers
#@+node:ekr.20171215074959.2: *5* checker.Assign & helpers
def before_Assign(self, node):
    
    s = self.format(node)
    if self.test_kind == 'test': print(s)
    if self.pass_n == 1:
        return
    self.stats.assignments += 1
    for target in node.targets:
        chain = self.get_chain(target)
        if len(chain) == 2:
            var1, var2 = chain
            assert isinstance(var1, ast.Name), repr(var1)
            assert isinstance(var2, str), repr(var2)
            name = var1.id
            if name == 'self':
                self.do_assn_to_self(node, name, var2)
            elif name in self.special_names_dict:
                self.do_assn_to_special(node, name, var2)
#@+node:ekr.20171215074959.4: *6* checker.do_assn_to_self
def do_assn_to_self(self, node, var1, var2):

    assert self.pass_n == 2
    assert var1 == 'self'
    class_name = self.class_name
    if not class_name:
        self.note(node, 'SKIP: no class name', self.format(node))
        return
    if class_name in self.special_class_names:
        # self.note(node, 'SKIP: not special', self.format(node))
        return
    d = self.classes.get(class_name)
    assert d is not None, class_name
    ivars = d.get('ivars')
    ivars[var2] = self.format(node.value)
    d['ivars'] = ivars
#@+node:ekr.20171215074959.3: *6* checker.do_assn_to_special
def do_assn_to_special(self, node, var1, var2):

    assert self.pass_n == 2
    assert var1 in self.special_names_dict, (repr(var1))
    class_name = self.class_name
    t = self.special_names_dict.get(var1)
    if not t:
        if 0: self.note(node, 'not special', var1, self.format(node).strip())
        return
    # Do not set members within the class itself.
    if t.kind == 'instance' and t.name == class_name:
        if 0: self.note(node, 'SKIP', var1, class_name)
        return
    # Resolve val, if possible.
    context = self.Type(
        'instance' if class_name else 'module',
        class_name or self.file_name,
    )
    self.recursion_count = 0
    value_s = self.format(node.value)
    resolved_type = self.resolve(node, value_s, context, trace=False)
    assert isinstance(resolved_type, self.Type), repr(resolved_type)
    if 0:
        self.note(node, f"context {context} : {value_s} ==> {resolved_type}")
    # Update var1's dict, not class_name's dict.
    d = self.classes.get(t.name)
    ivars = d.get('ivars')
    # tag:setter ivar1.ivar2 = Type
    ivars[var2] = resolved_type
    d['ivars'] = ivars
#@+node:ekr.20171215074959.5: *5* checker.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def before_Call(self, node):

    if self.test_kind == 'test':
        print(self.format(node))
    if self.pass_n == 1:
        return
    self.stats.calls += 1
    context = self.resolve_call(node)
    assert isinstance(context, self.Type)
    if context.kind != 'instance':
        return
    instance = self.classes.get(context.name)
    if not instance:
        return
    chain = self.get_chain(node.func)
    func = chain[-1]
    d = instance.get('methods')
    signature = d.get(func)
    if not signature:
        return
    if isinstance(signature, self.Type):
        pass # Already checked?
    else:
        args = [self.format(z) for z in node.args]
        signature = signature.split(',')
        self.check_signature(node, func, args, signature)
#@+node:ekr.20171215074959.7: *5* checker.ClassDef
def before_ClassDef(self, node):

    s = self.format(node, print_body=False)
    if self.test_kind == 'test': print(s)
    self.indent += 1
    self.context_stack.append(node)
    self.class_name = name = node.name
    if self.pass_n == 1:
        self.stats.classes += 1
        if name not in self.special_class_names:
            # tag:setter Init the class's dict.
            self.classes [name] = {'ivars': {}, 'methods': {}}

def after_ClassDef(self, node):

    self.indent -= 1
    if 0 and self.pass_n == 1:
        g.trace(node, self.show_stack())
        print(f"----- END class {self.class_name}. class dict...")
        g.printDict(self.classes.get(self.class_name))
    #
    # This code must execute in *both* passes.
    top = self.context_stack.pop()
    assert node == top, (node, top)
    # Set the class name
    self.class_name = None
    for node2 in reversed(self.context_stack):
        if isinstance(node2, ast.ClassDef):
            self.class_name = node2.name
            break
#@+node:ekr.20171215074959.9: *5* checker.FunctionDef
def before_FunctionDef(self, node):

    s = self.format(node, print_body=False)
    if self.test_kind == 'test': print(s)
    self.indent += 1
    self.context_stack.append(node)
    if self.pass_n == 1:
        self.stats.defs += 1
        if self.class_name not in self.special_class_names:
            if self.class_name in self.classes:
                the_class = self.classes.get(self.class_name)
                methods = the_class.get('methods')
                # tag:setter function-name=stringized-args
                methods [node.name] = self.format(node.args)
            # This is not an error.
            # else: g.error(node 'no class', node.name)

def after_FunctionDef(self, node):

    self.indent -= 1
    top = self.context_stack.pop()
    assert node == top, (node, top)
#@+node:ekr.20171216110107.1: *5* checker.get_chain
def get_chain(self,node):
    """Scan node for a chain of names."""
    chain, node1 = [], node
    while not isinstance(node, ast.Name):
        if isinstance(node, ast.Attribute):
            assert isinstance(node.attr, str), repr(node.attr)
            chain.append(node.attr)
            node = node.value
        else:
            name = node.__class__.__name__
            if name not in (
                'BoolOp', # c.config.getString('stylesheet') or ''.strip
                'Call', # c1.rootPosition().h = whatever
                'Dict', # {}.whatever.
                'Subscript', # d[x] = whatever
                'Str', # ''.join(), etc
                'Tuple', # (hPos,vPos) = self.getScroll()
            ):
                self.note(node1, '(get_chain) target %s:\n%s' % (
                    name, self.format(node1)))
            return []
    if isinstance(node, ast.Name):
        chain.append(node)
        return list(reversed(chain))
    return []
#@+node:ekr.20171215082648.1: *5* checker.show_stack
def show_stack(self):

    return g.listToString([
        '%15s %s' % (node.__class__.__name__, node.name)
            for node in self.context_stack
        ])
#@+node:ekr.20171212101613.1: *4* class CCStats
class CCStats:
    """
    A basic statistics class.  Use this way:
        
        stats = Stats()
        stats.classes += 1
        stats.defs += 1
        stats.report()
    """
    # Big sigh: define these to placate pylint.
    assignments = 0
    calls = 0
    check_signature = 0
    classes = 0
    defs = 0
    inference_fails = 0
    resolve = 0
    resolve_call = 0
    resolve_chain = 0
    resolve_ivar = 0
    sig_fail = 0
    sig_infer_fail = 0
    sig_infer_ok = 0
    sig_ok = 0
    sig_unknown = 0
        
    def report(self):
        aList = [z for z in dir(self) if not z.startswith('_') and z != 'report']
        n = max([len(z) for z in aList])
        for ivar in aList:
            print('%*s: %s' % (n, ivar, getattr(self, ivar)))
    
#@+node:ekr.20171214151001.1: *4* class CCTraverser (AstFullTraverser)
class CCTraverser (leoAst.AstFullTraverser):
    
    """A traverser class that *only* calls controller methods."""

    def __init__(self, controller):

        super().__init__()
        self.cc = controller
    
    def visit(self, node):
        """
        Visit a *single* ast node.
        Visitors are responsible for visiting children!
        """
        name = node.__class__.__name__
        assert isinstance(node, ast.AST), repr(node)
        before_method = getattr(self.cc, 'before_'+name, None)
        if before_method:
            before_method(node)
        do_method = getattr(self, 'do_'+name, None)
        do_method(node)
        after_method = getattr(self.cc, 'after_'+name, None)
        if after_method:
            after_method(node)
#@+node:ekr.20171209030742.1: *4* class Type
class Type:
    """A class to hold all type-related data."""

    kinds = ('error', 'class', 'func', 'instance', 'module', 'unknown')
    
    def __init__(self, kind, name, source=None, tag=None):

        assert kind in self.kinds, repr(kind)
        self.kind = kind
        self.name=name
        self.source = source
        self.tag = tag
        
    def __repr__(self):

        return f"<{self.kind}: {self.name}>"
        
    def __eq__(self, other):
        
        return self.kind == other.kind and self.name == other.name
#@+node:ekr.20160109102859.1: *3* class Context
class Context:
    """
    Context class (NEW)

    Represents a binding context: module, class or def.

    For any Ast context node N, N.cx is a reference to a Context object.
    """
    @others
#@+node:ekr.20160109103533.1: *4* Context.ctor
def __init__ (self, fn, kind, name, node, parent_context):
    """Ctor for Context class."""
    self.fn = fn
    self.kind = kind
    self.name = name
    self.node = node
    self.parent_context = parent_context
    # Name Data...
    self.defined_names = set()
    self.global_names = set()
    self.imported_names = set()
    self.nonlocal_names = set() # To do.
    self.st = {}
        # Keys are names seen in this context, values are defining contexts.
    self.referenced_names = set()
    # Node lists. Entries are Ast nodes...
    self.inner_contexts_list = []
    self.minor_contexts_list = []
    self.assignments_list = []
    self.calls_list = []
    self.classes_list = []
    self.defs_list = []
    self.expressions_list = []
    self.returns_list = []
    self.statements_list = []
    self.yields_list = []
    # Add this context to the inner context of the parent context.
    if parent_context:
        parent_context.inner_contexts_list.append(self)
#@+node:ekr.20160109134527.1: *4* Context.define_name
def define_name(self, name):
    """Define a name in this context."""
    self.defined_names.add(name)
    if name in self.referenced_names:
        self.referenced_names.remove(name)
#@+node:ekr.20160109143040.1: *4* Context.global_name
def global_name(self, name):
    """Handle a global name in this context."""
    self.global_names.add(name)
    # Not yet.
        # Python generates SyntaxWarnings when a name
        # is used before the corresponding global declarations.
        # We can make the same assumpution here:
        # give an *error* if an STE appears in this context for the name.
        # The error indicates that scope resolution will give the wrong result.
        # e = cx.st.d.get(name)
        # if e:
            # self.u.error(f"name {name!r} used prior to global declaration")
            # # Add the name to the global_names set in *this* context.
            # # cx.global_names.add(name)
        # # Regardless of error, bind the name in *this* context,
        # # using the STE from the module context.
        # cx.st.d[name] = module_e
#@+node:ekr.20160109144139.1: *4* Context.import_name
def import_name(self, module, name):

    if True and name == '*':
        g.trace('From x import * not ready yet')
    else:
        self.imported_names.add(name)
#@+node:ekr.20160109145526.1: *4* Context.reference_name
def reference_name(self, name):

    self.referenced_names.add(name)
#@+node:ekr.20160108105958.1: *3* class Pass1 (AstFullTraverser)
class Pass1 (leoAst.AstFullTraverser): # V2

    """ Pass1 does the following:

    1. Creates Context objects and injects them into the new_cx field of
       ast.Class, ast.FunctionDef and ast.Lambda nodes.

    2. Calls the following Context methods: cx.define/global/import/reference_name.
       These methods update lists used later to bind names to objects.
    """
    # pylint: disable=no-member
        # Stats class defines __setattr__
        # This is a known limitation of pylint.

    @others
#@+node:ekr.20160108105958.2: *4*  p1.ctor
def __init__(self, fn):

    super().__init__()
    self.fn = fn
    # Abbreviations...
    self.stats = Stats()
    self.u = ProjectUtils()
    self.format = leoAst.AstFormatter.format
    # Present context...
    self.context = None
    self.in_attr = False
        # True: traversing inner parts of an AST.Attribute tree.
    self.module_context = None
    self.parent = None
#@+node:ekr.20160108105958.3: *4*  p1.run (entry point)
def run (self,root):

    self.visit(root)
#@+node:ekr.20160109125654.1: *4*  p1.visit
def visit(self, node):
    """Visit a *single* ast node.  Visitors are responsible for visiting children!"""
    assert isinstance(node, ast.AST), node.__class__.__name__
    # Visit the children with the new parent.
    old_parent = self.parent
    self.parent = node
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    method(node)
    self.parent = old_parent
#@+node:ekr.20160108105958.11: *4* p1.visitors
#@+node:ekr.20160109134854.1: *5* Contexts
#@+node:ekr.20160108105958.8: *6* p1.def_args_helper
# arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults)

def def_args_helper (self,cx,node):

    assert self.kind(node) == 'arguments'
    self.visit_list(node.args)
    self.visit_list(node.defaults)
    for field in ('vararg','kwarg'): # node.field is a string.
        name = getattr(node,field,None)
        if name:
            # e = cx.st.define_name(name)
            self.stats.n_param_names += 1
#@+node:ekr.20160108105958.16: *6* p1.ClassDef
# ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)

def do_ClassDef (self,node):

    # pylint: disable=arguments-differ
    old_cx = self.context
    name = node.name
    # Define the class name in the old context.
    old_cx.define_name(name)
    # Visit bases in the old context.
    # bases = self.visit_list(node.bases)
    new_cx = Context(
        fn=None,
        kind='class',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx)
    # Visit the body in the new context.
    self.context = new_cx
    self.visit_list(node.body)
    self.context = old_cx
    # Stats.
    old_cx.classes_list.append(new_cx)
#@+node:ekr.20160108105958.19: *6* p1.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#    expr? returns)

def do_FunctionDef (self,node):
    # pylint: disable=arguments-differ
    # Define the function/method name in the old context.
    old_cx = self.context
    name = node.name
    old_cx.define_name(name)
    # Create the new context.
    new_cx = Context(
        fn=None,
        kind='def',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx) # Bug fix.
    # Visit in the new context...
    self.context = new_cx
    self.def_args_helper(new_cx,node.args)
    self.visit_list(node.body)
    self.context = old_cx
    # Stats
    old_cx.defs_list.append(new_cx)
#@+node:ekr.20160108105958.23: *6* p1.Interactive
def do_Interactive(self,node):

    assert False,'Interactive context not supported'
#@+node:ekr.20160108105958.24: *6* p1.Lambda
def do_Lambda (self,node):

    # Synthesize a lambda name in the old context.
    # This name must not conflict with split names of the form name@n.
    old_cx = self.context
    name = f"Lambda@@{self.stats.n_lambdas}"
    # Define a Context for the 'lambda' variables.
    new_cx = Context(
        fn=None,
        kind='lambda',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx)
    # Evaluate expression in the new context.
    self.context = new_cx
    self.def_args_helper(new_cx,node.args)
    self.visit(node.body)
    self.context = old_cx
    # Stats...
    self.stats.n_lambdas += 1
#@+node:ekr.20160108105958.26: *6* p1.Module
def do_Module (self,node):

    # Not yet: Get the module context from the global dict if possible.
    new_cx = Context(
        fn=self.fn,
        kind='module',
        name=None,
        node=node,
        parent_context=None)
    self.context = new_cx
    self.visit_list(node.body)
    self.context = None
#@+node:ekr.20160109135022.1: *5* Expressions
#@+node:ekr.20160108105958.13: *6* p1.Attribute (Revise)
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self,node):

    # Visit...
    # cx = self.context
    old_attr, self.in_attr = self.in_attr, True
    # ctx = self.kind(node.ctx)
    self.visit(node.value)
    # self.visit(node.ctx)
    self.in_attr = old_attr
    if not self.in_attr:
        base_node = node
        kind = self.kind(base_node)
        if kind in ('Builtin','Name'):
            # base_name = base_node.id
            pass
        elif kind in ('Dict','List','Num','Str','Tuple',):
            pass
        elif kind in ('BinOp','UnaryOp'):
            pass
        else:
            assert False,kind
    # Stats...
    self.stats.n_attributes += 1
#@+node:ekr.20160108105958.17: *6* p1.Expr
# Expr(expr value)

def do_Expr(self,node):

    # Visit...
    cx = self.context
    self.visit(node.value)
    # Stats...
    self.stats.n_expressions += 1
    cx.expressions_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.27: *6* p1.Name (REWRITE)
def do_Name(self,node):

    cx  = self.context
    ctx = self.kind(node.ctx)
    name = node.id
    # def_flag,ref_flag=False,False

    if ctx in ('AugLoad','AugStore','Load'):
        # Note: AugStore does *not* define the symbol.
        cx.reference_name(name)
        self.stats.n_load_names += 1
    elif ctx == 'Store':
        # if name not in cx.global_names:
        self.stats.n_store_names += 1
    elif ctx == 'Param':
        self.stats.n_param_refs += 1
    else:
        assert ctx == 'Del',ctx
        self.stats.n_del_names += 1
#@+node:ekr.20160109140648.1: *5* Imports
#@+node:ekr.20160108105958.21: *6* p1.Import
@ From Guido:

import x            -->  x = __import__('x')
import x as y       -->  y = __import__('x')
import x.y.z        -->  x = __import__('x.y.z')
import x.y.z as p   -->  p = __import__('x.y.z').y.z
@c

def do_Import(self,node):
    """
    Add the imported file to u.files_list if needed
    and create a context for the file.
    """
    cx = self.context
    cx.statements_list.append(node)
    # e_list, names = [],[]
    for fn,asname in self.get_import_names(node):
        self.resolve_import_name(fn)
        # Not yet.
        # # Important: do *not* analyze modules not in the files list.
        # if fn2:
            # mname = self.u.module_name(fn2)
            # if g.shortFileName(fn2) in self.u.files_list:
                # if mname not in self.u.module_names:
                    # self.u.module_names.append(mname)
            # def_name = asname or mname
            # names.append(def_name)
            # e = cx.st.define_name(def_name) # sets e.defined.
            # cx.imported_symbols_list.append(def_name)
            # e_list.append(e)

            # # Add the constant type to the list of types for the *variable*.
            # mod_cx = self.u.modules_dict.get(fn2) or LibraryModuleContext(self.u,fn2)
            # e.types_cache[''] = mod_cx.module_type
            # # self.u.stats.n_imports += 1

    # for e in e_list:
        # e.defs_list.append(node)
        # e.refs_list.append(node)
#@+node:ekr.20160108105958.22: *6* p1.ImportFrom
@ From Guido:

from p.q import x       -->  x = __import__('p.q', fromlist=['x']).x
from p.q import x as y  -->  y = __import__('p.q', fromlist=['x']).x
from ..x.y import z     -->  z = __import('x.y', level=2, fromlist=['z']).z

All these equivalences are still somewhat approximate; __import__
isn't looked up the way other variables are looked up (it is taken
from the current builtins), and if the getattr operation in the "from"
versions raises AttributeError that is translated into ImportError.

There's also a subtlety where "import x.y" implies that y must be a
submodule/subpackage of x, whereas in "from x import y" it may be
either a submodule/subpackage or a plain attribute (e.g. a class,
function or some other variable).
@c

def do_ImportFrom(self,node):
    """
    Add the imported file to u.files_list if needed
    and add the imported symbols to the *present* context.
    """
    cx = self.context
    cx.statements_list.append(node)
    self.resolve_import_name(node.module)
    for fn,asname in self.get_import_names(node):
        fn2 = asname or fn
        cx.import_name(fn2)
#@+node:ekr.20160108105958.9: *6* p1.get_import_names
def get_import_names (self,node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:

        if self.kind(ast2) == 'alias':
            data = ast2.name,ast2.asname
            result.append(data)
        else:
            g.trace('unsupported kind in Import.names list',self.kind(ast2))
    return result
#@+node:ekr.20160108105958.10: *6* p1.resolve_import_name
def resolve_import_name (self,spec):
    """Return the full path name corresponding to the import spec."""
    if not spec:
        return ''
    # This may not work for leading dots.
    aList = spec.split('.')
    path = None
    # paths = None
    name = 'no name'
    for name in aList:
        try:
            pass
            ### Not ready. Old code:
                # f,path,description = imp.find_module(name,paths)
                # if not path: break
                # paths = [path]
                # if f: f.close()
        except ImportError:
            # Important: imports can fail due to Python version.
            # Thus, such errors are not necessarily searious.
            path = None
            break
    if not path:
        return ''
    if path.endswith('.pyd'):
        return ''
    return path
#@+node:ekr.20160108105958.29: *5* Operators... To be deleted???
# operator = Add | BitAnd | BitOr | BitXor | Div
# FloorDiv | LShift | Mod | Mult | Pow | RShift | Sub |

def do_Add(self,node):       setattr(node,'op_name','+')
def do_BitAnd(self,node):    setattr(node,'op_name','&')
def do_BitOr(self,node):     setattr(node,'op_name','|')
def do_BitXor(self,node):    setattr(node,'op_name','^')
def do_Div(self,node):       setattr(node,'op_name','/')
def do_FloorDiv(self,node):  setattr(node,'op_name','//')
def do_LShift(self,node):    setattr(node,'op_name','<<')
def do_Mod(self,node):       setattr(node,'op_name','%')
def do_Mult(self,node):      setattr(node,'op_name','*')
def do_Pow(self,node):       setattr(node,'op_name','**')
def do_RShift(self,node):    setattr(node,'op_name','>>')
def do_Sub(self,node):       setattr(node,'op_name','-')

# boolop = And | Or
def do_And(self,node):       setattr(node,'op_name',' and ')
def do_Or(self,node):        setattr(node,'op_name',' or ')

# cmpop = Eq | Gt | GtE | In |
# Is | IsNot | Lt | LtE | NotEq | NotIn
def do_Eq(self,node):        setattr(node,'op_name','==')
def do_Gt(self,node):        setattr(node,'op_name','>')
def do_GtE(self,node):       setattr(node,'op_name','>=')
def do_In(self,node):        setattr(node,'op_name',' in ')
def do_Is(self,node):        setattr(node,'op_name',' is ')
def do_IsNot(self,node):     setattr(node,'op_name',' is not ')
def do_Lt(self,node):        setattr(node,'op_name','<')
def do_LtE(self,node):       setattr(node,'op_name','<=')
def do_NotEq(self,node):     setattr(node,'op_name','!=')
def do_NotIn(self,node):     setattr(node,'op_name',' not in ')

# unaryop = Invert | Not | UAdd | USub
def do_Invert(self,node):   setattr(node,'op_name','~')
def do_Not(self,node):      setattr(node,'op_name',' not ')
def do_UAdd(self,node):     setattr(node,'op_name','+')
def do_USub(self,node):     setattr(node,'op_name','-')
#@+node:ekr.20160109134929.1: *5* Minor contexts
#@+node:ekr.20160109130719.1: *6* p1.comprehension (to do)
# comprehension (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):

    # Visit...
    self.visit(node.target) # A name.
    self.visit(node.iter) # An attribute.
    for z in node.ifs:
        self.visit(z)
#@+node:ekr.20160108105958.18: *6* p1.For
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self,node):

    # Visit...
    cx = self.context
    self.visit(node.target)
    self.visit(node.iter)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    # Stats...
    self.stats.n_fors += 1
    cx.statements_list.append(node)
    cx.assignments_list.append(node)
#@+node:ekr.20160108105958.30: *6* p1.With
def do_With(self,node):

    # Visit...
    cx = self.context
    self.visit(node.context_expr)
    if node.optional_vars:
        self.visit(node.optional_vars)
    for z in node.body:
        self.visit(z)
    # Stats...
    self.stats.n_withs += 1
    cx.statements_list.append(node)
#@+node:ekr.20160109135003.1: *5* Statements
#@+node:ekr.20160108105958.12: *6* p1.Assign
def do_Assign(self,node):

    # Visit...
    for z in node.targets:
        self.visit(z)
    self.visit(node.value)
    # Stats...
    cx = self.context
    self.stats.n_assignments += 1
    cx.assignments_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.14: *6* p1.AugAssign
# AugAssign(expr target, operator op, expr value)

def do_AugAssign(self,node):

    # Visit...
    self.visit(node.target)
    self.visit(node.value)
    # Stats...
    cx = self.context
    self.stats.n_assignments += 1
    cx.assignments_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.15: *6* p1.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self,node):

    # Visit...
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
    # Stats...
    cx = self.context
    self.stats.n_calls += 1
    cx.calls_list.append(node)
#@+node:ekr.20160108105958.20: *6* p1.Global
def do_Global(self,node):

    # Visit
    cx = self.context
    for name in node.names:
        cx.global_name(name)
    # Stats...
    cx.statements_list.append(node)
    self.stats.n_globals += 1
#@+node:ekr.20160108105958.28: *6* p1.Return
def do_Return(self,node):

    # Visit...
    if node.value:
        self.visit(node.value)
    # Stats...
    self.stats.n_returns += 1
    cx = self.context
    cx.returns_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20150525123715.1: *3* class ProjectUtils
class ProjectUtils:
    """A class to compute the files in a project."""
    # To do: get project info from @data nodes.
    @others
#@+node:ekr.20150525123715.2: *4* pu.files_in_dir
def files_in_dir(self, theDir, recursive=True, extList=None, excludeDirs=None):
    """
    Return a list of all Python files in the directory.
    Include all descendants if recursiveFlag is True.
    Include all file types if extList is None.
    """
    # import glob
    import os
    # if extList is None: extList = ['.py']
    if excludeDirs is None: excludeDirs = []
    result = []
    if recursive:
        for root, dirs, files in os.walk(theDir):
            for z in files:
                fn = g.os_path_finalize_join(root, z)
                junk, ext = g.os_path_splitext(fn)
                if not extList or ext in extList:
                    result.append(fn)
            if excludeDirs and dirs:
                for z in dirs:
                    if z in excludeDirs:
                        dirs.remove(z)
    else:
        for ext in extList:
            result.extend(g.glob_glob(f"{theDir}.*{ext}"))
    return sorted(list(set(result)))
#@+node:ekr.20150525123715.3: *4* pu.get_project_directory
def get_project_directory(self, name):
    # Ignore everything after the first space.
    i = name.find(' ')
    if i > -1:
        name = name[: i].strip()
    leo_path, junk = g.os_path_split(__file__)
    d = {
        # Change these paths as required for your system.
        'coverage': r'C:\Python26\Lib\site-packages\coverage-3.5b1-py2.6-win32.egg\coverage',
        'leo': r'C:\leo.repo\leo-editor\leo\core',
        'lib2to3': r'C:\Python26\Lib\lib2to3',
        'pylint': r'C:\Python26\Lib\site-packages\pylint',
        'rope': r'C:\Python26\Lib\site-packages\rope-0.9.4-py2.6.egg\rope\base',
        'test': g.os_path_finalize_join(g.app.loadDir, '..', 'test-proj'),
    }
    dir_ = d.get(name.lower())
    if not dir_:
        g.trace(f"bad project name: {name}")
    if not g.os_path_exists(dir_):
        g.trace('directory not found:' % (dir_))
    return dir_ or ''
#@+node:ekr.20171213071416.1: *4* pu.leo_core_files
def leo_core_files(self):
    """Return all the files in Leo's core."""
    loadDir = g.app.loadDir
    # Compute directories.
    commands_dir = g.os_path_finalize_join(loadDir, '..', 'commands')
    plugins_dir = g.os_path_finalize_join(loadDir, '..', 'plugins')
    # Compute files.
    core_files = g.glob_glob('%s%s%s' % (loadDir, os.sep, '*.py'))
    for exclude in ['format-code.py',]:
        core_files = [z for z in core_files if not z.endswith(exclude)]
    command_files = g.glob_glob(f"{commands_dir}{os.sep}{'*.py'}")
    plugins_files = g.glob_glob(f"{plugins_dir}{os.sep}{'qt_*.py'}")
    # Compute the result.
    files = core_files + command_files + plugins_files
    files = [z for z in files if not z.endswith('__init__.py')]
    return files
#@+node:ekr.20150525123715.4: *4* pu.project_files
@nobeautify

def project_files(self, name, force_all=False):
    """Return a list of all files in the named project."""
    # Ignore everything after the first space.
    i = name.find(' ')
    if i > -1:
        name = name[: i].strip()
    leo_path, junk = g.os_path_split(__file__)
    if name == 'leo':
        # Get the leo files directly.
        return self.leo_core_files()
    # Import the appropriate module.
    try:
        m = importlib.import_module(name, name)
        theDir = g.os_path_dirname(m.__file__)
    except ImportError:
        g.trace('package not found', name)
        return []
    d = {
        'coverage': (['.py'], ['.bzr', 'htmlfiles']),
        'lib2to3':  (['.py'], ['tests']),
        'pylint':   (['.py'], ['.bzr', 'test']),
        'rope':     (['.py'], ['.bzr']),
    }
    data = d.get(name.lower())
    if not data:
        g.trace(f"bad project name: {name}")
        return []
    extList, excludeDirs = data
    files = self.files_in_dir(theDir,
        recursive=True,
        extList=extList,
        excludeDirs=excludeDirs,
    )
    if files:
        if g.app.runningAllUnitTests and len(files) > 1 and not force_all:
            return [files[0]]
    if not files:
        g.trace(f"no files found for {name} in {theDir}")
    if g.app.runningAllUnitTests and len(files) > 1 and not force_all:
        return [files[0]]
    return files
#@+node:ekr.20171213155537.1: *3* class NewShowData
class NewShowData:
    """The driver class for analysis project."""
    assigns_d = {}
    calls_d = {}
    classes_d = {}
    defs_d = {}
    returns_d = {}

    @others
#@+node:ekr.20171213160214.1: *4* sd.analyze
def analyze(self, fn, root):
    
    ast_d = {
        ast.Assign: self.assigns_d,
        ast.AugAssign: self.assigns_d,
        ast.Call: self.calls_d,
        ast.ClassDef: self.classes_d,
        ast.FunctionDef: self.defs_d,
        ast.Return: self.returns_d, 
    }
    fn = g.shortFileName(fn)
    for d in ast_d.values():
        d[fn] = []
    for node in ast.walk(root):
        d = ast_d.get(node.__class__)
        if d is not None:
            d[fn].append(self.format(node))
#@+node:ekr.20171214040822.1: *4* sd.dump
def dump(self, fn, root):
    
    suppress = [
        'arg', 'arguments', 'comprehension', 'keyword',
        'Attribute', 'BinOp', 'BoolOp', 'Dict', 'IfExp', 'Index',
        'Load', 'List', 'ListComp', 'Name', 'NameConstant', 'Num',
        'Slice', 'Store', 'Str', 'Subscript', 'Tuple', 'UnaryOp',
    ]
    # statements = ['Assign', 'AugAssign', 'Call', 'Expr', 'If', 'Return',]
    errors = set()
    fn = g.shortFileName(fn)
    for node in ast.walk(root):
        name = node.__class__.__name__
        if name not in suppress:
            try:
                print('%15s: %s' % (name, self.format(node,strip=False)))
            except AttributeError:
                errors.add(name)
    g.trace('errors', sorted(errors))
    # g.printList(sorted(errors))
#@+node:ekr.20171213163216.1: *4* sd.format
def format(self, node, strip=True):
    
    class Formatter(leoAst.AstFormatter):
        level = 0
    
    s = Formatter().visit(node)
    line1 = g.splitLines(s)[0]
    line1 = line1.strip() if strip else line1.rstrip()
    return g.truncate(line1, 80)
#@+node:ekr.20171213155537.3: *4* sd.run
def run(self, files, dump=False, show_results=True):
    """Process all files"""
    t1 = time.time()
    for fn in files:
        s, e = g.readFileIntoString(fn)
        if s:
            print('=====', g.shortFileName(fn))
            s1 = g.toEncodedString(s)
            root = ast.parse(s1, filename='before', mode='exec')
            if dump:
                self.dump(fn, root)
            else:
                self.analyze(fn, root)
        else:
            g.trace('skipped', g.shortFileName(fn))
    t2 = time.time()
    if show_results:
        self.show_results()
    g.trace('done: %s files in %4.1f sec.' % (len(files), (t2 - t1)))
#@+node:ekr.20171213155537.7: *4* sd.show_results
def show_results(self):
    """Print a summary of the test results."""
    table = (
        ('assignments', self.assigns_d),
        ('calls', self.calls_d),
        ('classes', self.classes_d),
        ('defs', self.defs_d),
        ('returns', self.returns_d),
    )
    for name, d in table:
        print(f"{name}...")
        g.printDict({key: sorted(set(d.get(key))) for key in d})
#@+node:ekr.20171213174732.1: *4* sd.visit
def visit(self, node, types):
    if isinstance(node, types):
        yield self.format(node)
#@+node:ekr.20150604164113.1: *3* class ShowData
class ShowData:
    """The driver class for analysis project."""
    @others
#@+node:ekr.20150604165500.1: *4*  ctor
def __init__(self, c):
    """Ctor for ShowData controller class."""
    self.c = c
    self.files = None
    # Data.
    self.assigns_d = {}
    self.calls_d = {}
    self.classes_d = {}
    self.context_stack = []
    self.defs_d = {}
    self.returns_d = {}
    # Statistics
    self.n_matches = 0
    self.n_undefined_calls = 0
    self.tot_lines = 0
    self.tot_s = 0
#@+node:ekr.20150604163903.1: *4* run & helpers
def run(self, files):
    """Process all files"""
    self.files = files
    t1 = time.time()
    for fn in files:
        s, e = g.readFileIntoString(fn)
        if s:
            self.tot_s += len(s)
            g.trace('%8s %s' % ("{:,}".format(len(s)), g.shortFileName(fn)))
                # Print len(s), with commas.
            # Fast, accurate:
            # 1.9 sec for parsing.
            # 2.5 sec for Null AstFullTraverer traversal.
            # 2.7 sec to generate all strings.
            # 3.8 sec to generate all reports.
            s1 = g.toEncodedString(s)
            self.tot_lines += len(g.splitLines(s))
                # Adds less than 0.1 sec.
            node = ast.parse(s1, filename='before', mode='exec')
            ShowDataTraverser(self, fn).visit(node)
            # elif 0: # Too slow, too clumsy: 3.3 sec for tokenizing
                # readlines = g.ReadLinesClass(s).next
                # for token5tuple in tokenize.generate_tokens(readlines):
                    # pass
            # else: # Inaccurate. 2.2 sec to generate all reports.
                # self.scan(fn, s)
        else:
            g.trace('skipped', g.shortFileName(fn))
    t2 = time.time()
        # Get the time exlusive of print time.
    self.show_results()
    g.trace('done: %4.1f sec.' % (t2 - t1))
#@+node:ekr.20150605054921.1: *4* scan & helpers (a prototype: no longer used)
if 0:
    # The excellent prototype code, fast, easy but inaccurate.
    # It was a roadmap for the ShowDataTraverser class.

    # Regex patterns (were defined in the ctor)
    r_class = r'class[ \t]+([a-z_A-Z][a-z_A-Z0-9]*).*:'
    r_def = r'def[ \t]+([a-z_A-Z][a-z_A-Z0-9]*)[ \t]*\((.*)\)'
    r_return = r'(return[ \t].*)$'
    r_call = r'([a-z_A-Z][a-z_A-Z0-9]*)[ \t]*\(([^)]*)\)'
    r_all = re.compile(r'|'.join([r_class, r_def, r_return, r_call,]))

    def scan(self, fn, s):
        lines = g.splitLines(s)
        self.tot_lines += len(lines)
        for i, s in enumerate(lines):
            m = re.search(self.r_all, s)
            if m and not s.startswith('@'):
                self.match(fn, i, m, s)
#@+node:ekr.20150605063318.1: *5* match
def match(self, fn, i, m, s):
    """Handle the next match."""
    self.n_matches += 1
    indent = g.skip_ws(s, 0)
    # Update the context and enter data.
    if g.match_word(s, indent, 'def'):
        self.update_context(fn, indent, 'def', s)
        for i, name in enumerate(m.groups()):
            if name:
                aList = self.defs_d.get(name, [])
                def_tuple = self.context_stack[: -1], s
                aList.append(def_tuple)
                self.defs_d[name] = aList
                break
    elif g.match_word(s, indent, 'class'):
        self.update_context(fn, indent, 'class', s)
        for i, name in enumerate(m.groups()):
            if name:
                aList = self.classes_d.get(name, [])
                class_tuple = self.context_stack[: -1], s
                aList.append(class_tuple)
                self.classes_d[name] = aList
    elif s.find('return') > -1:
        context, name = self.context_names()
        j = s.find('#')
        if j > -1: s = s[: j]
        s = s.strip()
        if s:
            aList = self.returns_d.get(name, [])
            return_tuple = context, s
            aList.append(return_tuple)
            self.returns_d[name] = aList
    else:
        # A call.
        for i, name in enumerate(m.groups()):
            if name:
                context2, context1 = self.context_names()
                j = s.find('#')
                if j > -1:
                    s = s[: j]
                s = s.strip().strip(',').strip()
                if s:
                    aList = self.calls_d.get(name, [])
                    call_tuple = context2, context1, s
                    aList.append(call_tuple)
                    self.calls_d[name] = aList
                break
#@+node:ekr.20150605074749.1: *5* update_context
def update_context(self, fn, indent, kind, s):
    """Update context info when a class or def is seen."""
    while self.context_stack:
        fn2, kind2, indent2, s2 = self.context_stack[-1]
        if indent <= indent2:
            self.context_stack.pop()
        else:
            break
    context_tuple = fn, kind, indent, s
    self.context_stack.append(context_tuple)
    self.context_indent = indent
#@+node:ekr.20150604164546.1: *4* show_results & helpers
def show_results(self):
    """Print a summary of the test results."""
    make = True
    multiple_only = False # True only show defs defined in more than one place.
    c = self.c
    result = ['@killcolor\n']
    for name in sorted(self.defs_d):
        aList = self.defs_d.get(name, [])
        if len(aList) > 1 or not multiple_only: # not name.startswith('__') and (
            self.show_defs(name, result)
            self.show_calls(name, result)
            self.show_returns(name, result)
    self.show_undefined_calls(result)
    # Put the result in a new node.
    format = (
        'files: %s lines: %s chars: %s classes: %s\n'
        'defs: %s calls: %s undefined calls: %s returns: %s'
    )
    summary = format % (
        # g.plural(self.files),
        len(self.files),
        "{:,}".format(self.tot_lines),
        "{:,}".format(self.tot_s),
        "{:,}".format(len(self.classes_d.keys())),
        "{:,}".format(len(self.defs_d.keys())),
        "{:,}".format(len(self.calls_d.keys())),
        "{:,}".format(self.n_undefined_calls),
        "{:,}".format(len(self.returns_d.keys())),
    )
    result.insert(1, summary)
    result.extend(['', summary])
    if c and make:
        last = c.lastTopLevel()
        p2 = last.insertAfter()
        p2.h = 'global signatures'
        p2.b = '\n'.join(result)
        c.redraw(p=p2)
    print(summary)
#@+node:ekr.20150605160218.1: *5* show_calls
def show_calls(self, name, result):
    aList = self.calls_d.get(name, [])
    if not aList:
        return
    result.extend(['', f"    {len(aList)} call{g.plural(aList)}..."])
    w = 0
    calls = sorted(set(aList))
    for call_tuple in calls:
        context2, context1, s = call_tuple
        w = max(w, len(context2 or '') + len(context1 or ''))
    for call_tuple in calls:
        context2, context1, s = call_tuple
        pad = w - (len(context2 or '') + len(context1 or ''))
        if context2:
            result.append('%s%s::%s: %s' % (
                ' ' * (8 + pad), context2, context1, s))
        else:
            result.append('%s%s: %s' % (
                ' ' * (10 + pad), context1, s))
#@+node:ekr.20150605155601.1: *5* show_defs
def show_defs(self, name, result):
    aList = self.defs_d.get(name, [])
    name_added = False
    w = 0
    # Calculate the width
    for def_tuple in aList:
        context_stack, s = def_tuple
        if context_stack:
            fn, kind, context_s = context_stack[-1]
            w = max(w, len(context_s))
    for def_tuple in aList:
        context_stack, s = def_tuple
        if not name_added:
            name_added = True
            result.append('\n%s' % name)
            result.append(f"    {len(aList)} definition{g.plural(aList)}...")
        if context_stack:
            fn, kind, context_s = context_stack[-1]
            def_s = s.strip()
            pad = w - len(context_s)
            result.append('%s%s: %s' % (' ' * (8 + pad), context_s, def_s))
        else:
            result.append('%s%s' % (' ' * 4, s.strip()))
#@+node:ekr.20150605160341.1: *5* show_returns
def show_returns(self, name, result):
    aList = self.returns_d.get(name, [])
    if not aList:
        return
    result.extend(['', f"    {len(aList)} return{g.plural(aList)}..."])
    w, returns = 0, sorted(set(aList))
    for returns_tuple in returns:
        context, s = returns_tuple
        w = max(w, len(context or ''))
    for returns_tuple in returns:
        context, s = returns_tuple
        pad = w - len(context)
        result.append('%s%s: %s' % (' ' * (8 + pad), context, s))
#@+node:ekr.20150606092147.1: *5* show_undefined_calls
def show_undefined_calls(self, result):
    """Show all calls to undefined functions."""
    call_tuples = []
    for s in self.calls_d:
        i = 0
        while True:
            progress = i
            j = s.find('.', i)
            if j == -1:
                name = s[i:].strip()
                call_tuple = name, s
                call_tuples.append(call_tuple)
                break
            else:
                i = j + 1
            assert progress < i
    undef = []
    for call_tuple in call_tuples:
        name, s = call_tuple
        if name not in self.defs_d:
            undef.append(call_tuple)
    undef = list(set(undef))
    result.extend(['', f"{len(undef)} undefined call{g.plural(undef)}..."])
    self.n_undefined_calls = len(undef)
    # Merge all the calls for name.
    # There may be several with different s values.
    results_d = {}
    for undef_tuple in undef:
        name, s = undef_tuple
        calls = self.calls_d.get(s, [])
        aList = results_d.get(name, [])
        for call_tuple in calls:
            aList.append(call_tuple)
        results_d[name] = aList
    # Print the final results.
    for name in sorted(results_d):
        calls = results_d.get(name)
        result.extend(['', f"{name} {len(calls)} call{g.plural(calls)}..."])
        w = 0
        for call_tuple in calls:
            context2, context1, s = call_tuple
            if context2:
                w = max(w, 2 + len(context2) + len(context1))
            else:
                w = max(w, len(context1))
        for call_tuple in calls:
            context2, context1, s = call_tuple
            pad = w - (len(context2) + len(context1))
            if context2:
                result.append('%s%s::%s: %s' % (
                    ' ' * (2 + pad), context2, context1, s))
            else:
                result.append('%s%s: %s' % (
                    ' ' * (2 + pad), context1, s))
#@+node:ekr.20150605140911.1: *4* context_names
def context_names(self):
    """Return the present context name."""
    if self.context_stack:
        result = []
        for stack_i in -1, -2:
            try:
                fn, kind, indent, s = self.context_stack[stack_i]
            except IndexError:
                result.append('')
                break
            s = s.strip()
            assert kind in ('class', 'def'), kind
            i = g.skip_ws(s, 0)
            i += len(kind)
            i = g.skip_ws(s, i)
            j = g.skip_c_id(s, i)
            result.append(s[i: j])
        return reversed(result)
    return ['', '']
#@+node:ekr.20150606024455.1: *3* class ShowDataTraverser (AstFullTraverser)
class ShowDataTraverser(leoAst.AstFullTraverser):
    """
    Add data about classes, defs, returns and calls to controller's
    dictionaries.
    """

    def __init__(self, controller, fn):
        """Ctor for ShopDataTraverser class."""
        super().__init__()
        module_tuple = g.shortFileName(fn), 'module', g.shortFileName(fn)
            # fn, kind, s.
        self.context_stack = [module_tuple]
        self.controller = controller
        self.fn = g.shortFileName(fn)
        self.formatter = leoAst.AstFormatter()
            # leoAst.AstPatternFormatter()
    @others
#@+node:ekr.20150609053332.1: *4* sd.Helpers
#@+node:ekr.20150606035006.1: *5* sd.context_names
def context_names(self):
    """Return the present context names."""
    result = []
    n = len(self.context_stack)
    for i in n - 1, n - 2:
        if i >= 0:
            fn, kind, s = self.context_stack[i]
            assert kind in ('class', 'def', 'module'), kind
            if kind == 'module':
                result.append(s.strip())
            else:
                # Append the name following the class or def.
                i = g.skip_ws(s, 0)
                i += len(kind)
                i = g.skip_ws(s, i)
                j = g.skip_c_id(s, i)
                result.append(s[i: j])
        else:
            result.append('')
            break
    return reversed(result)
#@+node:ekr.20150609053010.1: *5* sd.format
def format(self, node, level, *args, **kwargs):
    """Return the formatted version of an Ast Node."""
    return self.formatter.format(node, level, *args, **kwargs).strip()
#@+node:ekr.20150606024455.62: *4* sd.visit
def visit(self, node):
    """
    Visit a *single* ast node. Visitors must visit their children
    explicitly.
    """
    method = getattr(self, 'do_' + node.__class__.__name__)
    method(node)

def visit_children(self, node):
    """Override to ensure this method is never called."""
    assert False, 'must visit children explicitly'
#@+node:ekr.20150609052952.1: *4* sd.Visitors
#@+node:ekr.20150607200422.1: *5* sd.Assign
def do_Assign(self, node):
    """Handle an assignment statement: Assign(expr* targets, expr value)"""
    value = self.format(self.visit(node.value), self.level)
    assign_tuples = []
    for target in node.targets:
        target = self.format(self.visit(target), self.level)
        s = '%s=%s' % (target, value)
        context2, context1 = self.context_names()
        assign_tuple = context2, context1, s
        assign_tuples.append(assign_tuple)
        aList = self.controller.assigns_d.get(target, [])
        aList.extend(assign_tuples)
        self.controller.calls_d[target] = aList
#@+node:ekr.20150607200439.1: *5* sd.AugAssign
def do_AugAssign(self, node):
    """
    Handle an augmented assignement:
    AugAssign(expr target, operator op, expr value).
    """
    target = self.format(self.visit(node.target), self.level)
    s = '%s=%s' % (target, self.format(self.visit(node.value), self.level))
    context2, context1 = self.context_names()
    assign_tuple = context2, context1, s
    aList = self.controller.assigns_d.get(target, [])
    aList.append(assign_tuple)
    self.controller.calls_d[target] = aList
#@+node:ekr.20150606024455.16: *5* sd.Call
def do_Call(self, node):
    """
    Handle a call statement:
    Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)
    """
    # Update data.
    s = self.format(node, self.level)
    name = self.format(node.func, self.level)
    context2, context1 = self.context_names()
    call_tuple = context2, context1, s
    aList = self.controller.calls_d.get(name, [])
    aList.append(call_tuple)
    self.controller.calls_d[name] = aList
    # Visit.
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
#@+node:ekr.20150606024455.3: *5* sd.ClassDef
def do_ClassDef(self, node):
    """
    Handle a class defintion:
    ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
    """
    # pylint: disable=arguments-differ
    # Format.
    if node.bases:
        bases = [self.format(z, self.level) for z in node.bases]
        s = 'class %s(%s):' % (node.name, ','.join(bases))
    else:
        s = 'class %s:' % node.name
    # Enter the new context.
    context_tuple = self.fn, 'class', s
    self.context_stack.append(context_tuple)
    # Update data.
    class_tuple = self.context_stack[: -1], s
    aList = self.controller.classes_d.get(node.name, [])
    aList.append(class_tuple)
    self.controller.classes_d[node.name] = aList
    # Visit.
    for z in node.bases:
        self.visit(z)
    for z in node.body:
        self.visit(z)
    for z in node.decorator_list:
        self.visit(z)
    # Leave the context.
    self.context_stack.pop()
#@+node:ekr.20150606024455.4: *5* sd.FunctionDef
def do_FunctionDef(self, node):
    """
    Visit a function defintion:
    FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
    """
    # pylint: disable=arguments-differ
    # Format.
    args = self.format(node.args, self.level) if node.args else ''
    s = 'def %s(%s):' % (node.name, args)
    # Enter the new context.
    context_tuple = self.fn, 'def', s
    self.context_stack.append(context_tuple)
    # Update data.
    def_tuple = self.context_stack[: -1], s
    aList = self.controller.defs_d.get(node.name, [])
    aList.append(def_tuple)
    self.controller.defs_d[node.name] = aList
    # Visit.
    for z in node.decorator_list:
        self.visit(z)
    self.visit(node.args)
    for z in node.body:
        self.visit(z)
    # Leave the context.
    self.context_stack.pop()
#@+node:ekr.20150606024455.55: *5* sd.Return
def do_Return(self, node):
    """Handle a 'return' statement: Return(expr? value)"""
    # Update data.
    s = self.format(node, self.level)
    context, name = self.context_names()
    aList = self.controller.returns_d.get(name, [])
    return_tuple = context, s
    aList.append(return_tuple)
    self.controller.returns_d[name] = aList
    # Visit.
    if node.value:
        self.visit(node.value)
#@+node:ekr.20171211163833.1: *3* class Stats
class Stats:
    """
    A basic statistics class.  Use this way:
        
        stats = Stats()
        stats.classes += 1
        stats.defs += 1
        stats.report()
    """

    d = {}
    
    def __getattr__(self, name):
        return self.d.get(name, 0)
        
    def __setattr__(self, name, val):
        self.d[name] = val
        
    def report(self):
        if self.d:
            n = max([len(key) for key in self.d])
            for key, val in sorted(self.d.items()):
                print('%*s: %s' % (n, key, val))
        else:
            print('no stats')
#@+node:ekr.20171211055756.1: *3* checkConventions (checkerCommands.py)
@g.command('check-conventions')
@g.command('cc')
def checkConventions(event):
    """Experimental script to test Leo's convensions."""
    c = event.get('c')
    if c:
        if c.changed: c.save()
        import importlib
        import leo.core.leoCheck as leoCheck
        importlib.reload(leoCheck)
        leoCheck.ConventionChecker(c).check()
#@+node:ekr.20210809085711.1: ** From leoGlobals.py
#@+node:ekr.20210809093522.1: *3* g.adjustTripleString
def adjustTripleString(s, tab_width):
    """Remove leading indentation from a triple-quoted string.

    This works around the fact that Leo nodes can't represent underindented strings.
    """
    # Compute the minimum leading whitespace of all non-blank lines.
    lines = g.splitLines(s)
    first, w = True, 0
    for line in lines:
        if line.strip():
            lws = g.get_leading_ws(line)
            # The sign of w2 does not matter.
            w2 = abs(g.computeWidth(lws, tab_width))
            if w2 == 0:
                return s
            if first or w2 < w:
                w = w2
                first = False
    if w == 0:
        return s
    # Remove the leading whitespace.
    result = [g.removeLeadingWhitespace(line, w, tab_width) for line in lines]
    return ''.join(result)
#@+node:ekr.20050211120242.2: *3* g.removeExtraLws
def removeExtraLws(s: str, tab_width):
    """
    Remove extra indentation from one or more lines.

    Warning: used by getScript.
    """
    lines = g.splitLines(s)
    # Find the first non-blank line and compute w, the width of its leading whitespace.
    for line in lines:
        if line.strip():
            lws = g.get_leading_ws(line)
            w = g.computeWidth(lws, tab_width)
            break
    else: return s
    # Remove the leading whitespace.
    result = [g.removeLeadingWhitespace(line, w, tab_width) for line in lines]
    return ''.join(result)
#@+node:ekr.20210327062138.1: ** From leoRst.py
#@+node:ekr.20120219194520.10444: *3* html parser classes
# Perhaps mod_http used to use these.
#@+node:ekr.20120219194520.10445: *4*  class LinkAnchorParserClass (HTMLParser)
class LinkAnchorParserClass(HTMLParser.HTMLParser):
    """
    The base class to recognize anchors and links in HTML documents. A
    special marker is the "node_marker" which marks the border between node
    and the next.

    The parser classes are used to construct the html code for nodes.

    The algorithm has two phases:
    - Phase 1 AnchorHtmlParserClass: gets the html code for each node.
    - Phase 2 LinkHtmlParserClass: finds all links and checks whethr
      these links need to be modified.
    """
    @others
#@+node:ekr.20120219194520.10446: *5* __init__
def __init__(self, rst, p):
    """Ctor for the LinkAnchorParserClass class."""
    super().__init__()
    self.rst = rst
    self.p = p.copy()
    # Set ivars from options.
    # This works only if we don't change nodes!
    self.node_begin_marker = rst.getOption(p, 'node_begin_marker')
    self.clear_http_attributes = rst.getOption(p, 'clear_http_attributes')
    self.current_file = rst.outputFileName
#@+node:ekr.20120219194520.10447: *5* is_anchor
def is_anchor(self, tag, attrs):
    """
    Check if the current tag is an anchor.
    Returns *all* anchors.
    Works with docutils 0.4
    """
    if tag == 'a':
        return True
    if self.is_node_marker(attrs):
        return True
    return tag == "span"
#@+node:ekr.20120219194520.10448: *5* is_link
def is_link(self, tag, attrs):
    """Return True if tag, attrs is represents a link."""
    if tag == 'a':
        return 'href' in dict(attrs)
    return False
#@+node:ekr.20120219194520.10449: *5* is_node_marker
def is_node_marker(self, attrs):
    """
    Return the name of the anchor, if this is an anchor for the beginning of a node,
    False otherwise.
    """
    d = dict(attrs)
    if d.get('id', '').startswith(self.node_begin_marker):
        return d['id']
    return False
#@+node:ekr.20120219194520.10450: *4* class HtmlParserClass (LinkAnchorParserClass)
class HtmlParserClass(LinkAnchorParserClass):
    """
    The responsibility of the html parser is:
        1. Find out which html code belongs to which node.
        2. Keep a stack of open tags which apply to the current node.
        3. Keep a list of tags which should be included in the nodes, even
           though they might be closed.
           The <style> tag is one example of that.

    Later, we have to relocate inter-file links: if a reference to another location
    is in a file, we must change the link.

    """
    @others
#@+node:ekr.20120219194520.10451: *5* HtmlParserClass.__init__
def __init__(self, rst, p):
    """Ctor for the HtmlParserClass class."""
    super().__init__(rst, p)
    self.stack = None
        # The stack contains lists of the form:
        # [text1, text2, previous].
        # text1 is the opening tag
        # text2 is the closing tag
        # previous points to the previous stack element
    self.node_marker_stack = []
        # self.node_marker_stack.pop() returns True for a closing tag if
        # the opening tag identified an anchor belonging to a VNode.
    self.node_code = []
        # Accumulated html code.
        # Once the hmtl code is assigned a VNode, it is deleted here.
    self.deleted_lines = 0
        # Number of lines deleted in self.node_code
    self.endpos_pending = False
        # Do not include self.node_code[0:self.endpos_pending] in the html code.
    self.last_position = None
        # Last position; we must attach html code to this node.
    self.last_marker = None
#@+node:ekr.20120219194520.10452: *5* HtmlParserClass.handle_starttag
def handle_starttag(self, tag, attrs):
    """
    1. Find out if the current tag is an achor.
    2. If it is an anchor, we check if this anchor marks the beginning of a new
       node
    3. If a new node begins, then we might have to store html code for the previous
       node.
    4. In any case, put the new tag on the stack.
    """
    is_node_marker = False
    if self.is_anchor(tag, attrs) and self.is_node_marker(attrs):
        is_node_marker = self.is_node_marker(attrs)
        line, column = self.getpos()
        if self.last_position:
            lines = self.node_code[:]
            lines[0] = lines[0][self.startpos:]
            del lines[line - self.deleted_lines - 1 :]
            mod_http.get_http_attribute(self.last_position).extend(lines)
            << trace the unknownAttribute >>
        if self.deleted_lines < line - 1:
            del self.node_code[: line - 1 - self.deleted_lines]
            self.deleted_lines = line - 1
            self.endpos_pending = True
    starttag = self.get_starttag_text()
    self.stack = [starttag, None, self.stack]
    self.node_marker_stack.append(is_node_marker)
#@+node:ekr.20120219194520.10453: *6* << trace the unknownAttribute >>
if 0:
    g.pr("rst3: unknownAttributes[self.http_attributename]")
    g.pr("For:", self.last_position)
    pprint.pprint(mod_http.get_http_attribute(self.last_position))
#@+node:ekr.20120219194520.10454: *5* HtmlParserClass.handle_endtag
def handle_endtag(self, tag):
    """
    1. Set the second element of the current top of stack.
    2. If this is the end tag for an anchor for a node,
       store the current stack for that node.
    """
    self.stack[1] = "</" + tag + ">"
    if self.endpos_pending:
        line, column = self.getpos()
        self.startpos = self.node_code[0].find(">", column) + 1
        self.endpos_pending = False
    is_node_marker = self.node_marker_stack.pop()
    if is_node_marker and not self.clear_http_attributes:
        self.last_position = self.rst.http_map[is_node_marker]
        if is_node_marker != self.last_marker:
            # if bwm_file: print >> bwm_file, "Handle endtag:", is_node_marker, self.stack
            mod_http.set_http_attribute(
                self.rst.http_map[is_node_marker], self.stack)
            self.last_marker = is_node_marker
            # bwm: last_marker is not needed?
    self.stack = self.stack[2]
#@+node:ekr.20120219194520.10455: *5* HtmlParserClass.feed
def feed(self, line):
    # pylint: disable=arguments-differ
    self.node_code.append(line)
    HTMLParser.HTMLParser.feed(self, line)  # Call the base class's feed().
#@+node:ekr.20120219194520.10456: *4* class AnchorHtmlParserClass (LinkAnchorParserClass)
class AnchorHtmlParserClass(LinkAnchorParserClass):
    """
    This htmlparser does the first step of relocating: finding all the
    anchors within the html nodes.

    Each anchor is mapped to a tuple: (current_file, position).

    Filters out markers which mark the beginning of the html code for a node.
    """
    @others
#@+node:ekr.20120219194520.10457: *5*  __init__
def __init__(self, rst, p):
    """Ctor for the AnchorHtmlParserClass class."""
    super().__init__(rst, p)
    self.p = p.copy()
    self.anchor_map = rst.anchor_map
#@+node:ekr.20120219194520.10458: *5* handle_starttag
def handle_starttag(self, tag, attrs):
    """
    1. Find out if the current tag is an achor.
    2. If the current tag is an anchor, update the mapping;
         anchor -> (filename, p)
    """
    if not self.is_anchor(tag, attrs):
        return
    if self.current_file not in self.anchor_map:
        self.anchor_map[self.current_file] = (self.current_file, self.p)
        simple_name = g.os_path_split(self.current_file)[1]
        self.anchor_map[simple_name] = self.anchor_map[self.current_file]
        # if bwm_file:
        #   print >> bwm_file, "anchor(1): current_file:",
        #   self.current_file, "position:", self.p, "Simple name:", simple_name
        # Not sure what to do here, exactly. Do I need to manipulate
        # the pathname?
    for name, value in attrs:
        if name == 'name' or tag == 'span' and name == 'id':
            if not value.startswith(self.node_begin_marker):
                # if bwm_file: print >> bwm_file, "anchor(2):", value, self.p
                self.anchor_map[value] = (self.current_file, self.p.copy())
#@+node:ekr.20120219194520.10459: *4* class LinkHtmlParserClass (LinkAnchorParserClass)
class LinkHtmlparserClass(LinkAnchorParserClass):
    """This html parser does the second step of relocating links:
    1. It scans the html code for links.
    2. If there is a link which links to a previously processed file
       then this link is changed so that it now refers to the node.
    """
    @others
#@+node:ekr.20120219194520.10460: *5* __init__
def __init__(self, rst, p):
    """Ctor for the LinkHtmlParserClass class."""
    super().__init__(rst, p)
    self.anchor_map = rst.anchor_map
    self.replacements = []
#@+node:ekr.20120219194520.10461: *5* handle_starttag
def handle_starttag(self, tag, attrs):
    """
    1. Find out if the current tag is an achor.
    2. If the current tag is an anchor, update the mapping;
         anchor -> p
        Update the list of replacements for the document.
    """
    if not self.is_link(tag, attrs):
        return
    marker = self.node_begin_marker
    for name, value in attrs:
        if name == 'href':
            href = value
            href_parts = href.split("#")
            if len(href_parts) == 1:
                href_a = href_parts[0]
            else:
                href_a = href_parts[1]
            if not href_a.startswith(marker):
                if href_a in self.anchor_map:
                    href_file, href_node = self.anchor_map[href_a]
                    http_node_ref = mod_http.node_reference(href_node)
                    line, column = self.getpos()
                    self.replacements.append(
                        (line, column, href, href_file, http_node_ref))
#@+node:ekr.20120219194520.10462: *5* get_replacements
def get_replacements(self):
    return self.replacements
#@+node:ekr.20210328100631.1: *3* insert-* commands from #1843 (move to attic)
# These will never
#@+node:ekr.20210328100510.2: *4* rst.insertBody
def insertBody(self, p):
    """
    Handle @rst-insert-body.
    
    Insert p2.b into the output, where p2 has the given gnx.
    """
    c = self.c
    lines = g.splitLines(p.b.strip())
    gnx = lines and lines[0].strip()
    p2 = self.findGnx(c, gnx)
    if p2:
        self.write(f"\n\n{p2.b.strip()}\n\n")
    else:
        g.es_print(f"gnx not found: {gnx!r} in {p.h}")
#@+node:ekr.20210328100510.3: *4* rst.insertHead
def insertHead(self, p):
    """
    Handle @rst-insert-head.
    
    Insert p2.h into the output, where p2 has the given gnx.
    """
    c = self.c
    lines = g.splitLines(p.b.strip())
    gnx = lines and lines[0].strip()
    p2 = self.findGnx(c, gnx)
    if p2:
        self.write(f"\n\n{p2.h}\n\n")
    else:
        g.es_print(f"gnx not found: {gnx!r} in {p.h}")
#@+node:ekr.20210328100510.4: *4* rst.insertTree
def insertTree(self, p):
    """
    Handle @rst-insert-tree.
    
    Insert a representation of the tree of headlines.
    """
    c = self.c
    lines = g.splitLines(p.b.strip())
    gnx = lines and lines[0].strip()
    p2 = self.findGnx(c, gnx)
    if p2:
        level0 = p.level()
        tree = [f"    {' '*2*(z.level()-level0)}- {z.h}\n" for z in p2.self_and_subtree()]
        self.write(f"\n\n::\n\n{''.join(tree)}\n\n")
    else:
        g.es_print(f"gnx not found: {gnx!r} in {p.h}")
#@+node:ekr.20210712134630.1: ** From leoPersistence.py
@g.command('at-file-to-at-auto')
def at_file_to_at_auto_command(event):
    c = event.get('c')
    if c and c.persistenceController:
        c.persistenceController.convert_at_file_to_at_auto(c.p)
#@+node:ekr.20210717043934.1: ** From qt_tree.pyi
#@+node:ekr.20110605121601.17900: *3* qtree.OnPopup & allies
def OnPopup(self, p, event):
    """Handle right-clicks in the outline.

    This is *not* an event handler: it is called from other event handlers."""
    # Note: "headrclick" hooks handled by VNode callback routine.
    if event:
        c = self.c
        c.setLog()
        if not g.doHook("create-popup-menu", c=c, p=p, event=event):
            self.createPopupMenu(event)
        if not g.doHook("enable-popup-menu-items", c=c, p=p, event=event):
            self.enablePopupMenuItems(p, event)
        if not g.doHook("show-popup-menu", c=c, p=p, event=event):
            self.showPopupMenu(event)
    return "break"
#@+node:ekr.20110605121601.17901: *4* qtree.OnPopupFocusLost
@language rest
@
On Linux we must do something special to make the popup menu "unpost" if the
mouse is clicked elsewhere. So we have to catch the <FocusOut> event and
explicitly unpost. In order to process the <FocusOut> event, we need to be able
to find the reference to the popup window again, so this needs to be an
attribute of the tree object; hence, "self.popupMenu".

Aside: though Qt tries to be muli-platform, the interaction with different
window managers does cause small differences that will need to be compensated by
system specific application code. :-(
@c
@language python

# 20-SEP-2002 DTHEIN: This event handler is only needed for Linux.

def OnPopupFocusLost(self, event=None):
    # self.popupMenu.unpost()
    pass
#@+node:ekr.20110605121601.17902: *4* qtree.createPopupMenu
def createPopupMenu(self, event):
    """This might be a placeholder for plugins.  Or not :-)"""
#@+node:ekr.20110605121601.17903: *4* qtree.enablePopupMenuItems
def enablePopupMenuItems(self, p, event):
    """Enable and disable items in the popup menu."""
#@+node:ekr.20110605121601.17904: *4* qtree.showPopupMenu
def showPopupMenu(self, event):
    """Show a popup menu."""
#@+node:ekr.20210718034819.1: ** From LeoDocs.leo: 
The section covers complex options arising from two equivalent problems:

- How to generate documentation from computer source code in a Leo outline.
- How to embed documentation in computer source code in a Leo outline.

*Please stop reading now if these problems don't interest you!*
#@+node:ekr.20210822060917.1: *3* rstplugin3.html: Advanced topics
#@+node:ekr.20210718034819.2: *4* Modes
The rst3 command supports three different modes.

``rst mode``
    The default mode, as discussed in the Tutorial. The rst3 command treats body text as rST (or Sphinx) markup.
    
``code mode``
    The rst3 command treats body text as computer source code, generating the appropriate rST or Sphinx markup. Code mode is inherently complex. It supports *many* options.
    
``doc-only mode``
    The rst3 command outputs only regular doc parts and @ @rst-markup doc parts. Headlines create section in doc_only mode only if:

1. The node contains a doc part or

2. The show_organizer_nodes option is in effect.

The code_mode and doc_only_mode options determine the mode as follows:

``code_mode=False; doc_only_mode=False (the default)``
    Enters rst mode.
    
``code_mode=False; doc_only_mode=True``
    Enters doc_only mode.

``code_mode=True; (doc_only_mode ignored)``
    Enters code mode.
#@+node:ekr.20210718034819.3: *4* Code mode options
The following options have effect only in code mode.

.. glossary::
    :sorted:

``number_code_lines (default: True)``
    Controls whether to number code lines in code mode. *This option has no effect in rst mode*.

``show_leo_directives (default: True)``
    True: include Leo directives False: ignore Leo directives.

``show_markup_doc_parts (default: False)``
    True: include markup doc parts. False: ignore markup doc parts.

``show_options_doc_parts (default: False)``
    True: include options doc parts. False: ignore options doc parts.

``show_doc_parts_as_paragraphs (default: False)``
    True: Move doc parts outside of the code-block directive. False: Show doc parts in the code-block directive.
    
    **Cool**: Any rST markup in doc parts included as the result of this option will be rendered properly.

``show_options_nodes (default: False)``
    True: show @rst-options nodes. False: Ignore @
#@+node:ekr.20210718034819.4: *4* Rst mode options
The following option has effect only in rst mode.

.. glossary::

``show_doc_parts_in_rst_mode [True,False or class names] (default: True)``
    This option is most useful for rst documents which are not computer code. It allows you to use doc parts to make comments on the draft document which are either excluded from the output or formatted in a way that highlights their nature as comments rather than content. For example, you're writing a book, and you want to use a doc part at the top of a section to remind yourself "need to explain how Ted got to Sally's". Note: you may need to add CSS to have them formatted differently.

    The option can be `True`, `False`, or one or more class names.
    
    **True**: Treat the entire doc part from the opening '@' to the closing '@c
    as normal markup.
    
    **False**: Remove the doc part.

    **class names**: Process the contents of the doc part as it if were in an rst `container` directive. For example::

         @ @rst-options
         show_doc_parts_in_rst_mode = notes literal
         @c

This would wrap the doc part contents in the output in a div with classes "container notes literal". Furthermore, if one of the class names is ``literal``, then the doc part content will be output as a literal block wrapped in a container as described above. This allows you to use text which is not valid rst as rough notes for annotating a draft document.
#@+node:ekr.20210718034819.5: *4* The code-block directive
The rst3 command defines a code-block rST directive. The primary purpose of this directive is to show formatted source code.

In rst mode you can insert the code-block directive like any other rST markup. The rst3 command generates code-block directives automatically in code mode. This directive takes one argument, a language name.  Like this::

    .. code-block:: Python

        import leo.core.leoPlugins as leoPlugins
        import leo.core.leoGlobals as g

The output looks like this::

    import leo.core.leoPlugins as leoPlugins
    import leo.core.leoGlobals as g

.. _`Scripting Tutorial`:   tutorial-scripting.html

See the `Scripting Tutorial`_ for many examples of how to use code-blocks.
#@+node:ekr.20210822060938.1: *3* \@test and @suite nodes
#@+node:ekr.20210822055951.1: *4* \@test nodes
**@test nodes** create unit tests. @test nodes automatically convert the body to a subclass of unittest.TestCase. Run these tests with one of Leo's ``run-unit-test-`` commands. ``<Alt-X>run<tab>`` gives the full list. Here one of Leo's actual unit tests::

    @test c.positionExists for all nodes # In the headline

    for p in c.all_positions():
        assert c.positionExists(p)
    
Within @test nodes, c, g, and p are predefined as usual. Also, **self** is the instance of unittest.TestCase created by the @test node. For example::
  
    self.assertTrue(g)
    
For more details, see `Leo's unit-testing reference <unitTesting.html>`_.
#@+node:ekr.20210822060903.1: *4* @rst html\unitTesting.html
#####################
Unit testing with Leo
#####################

.. _`run Leo in a console window`: installing.html#running-leo-from-a-console-window

This chapter describes how you can execute Python unit test from within Leo
outlines.

Leo's **unit test commands** run the unit tests created by @test and @suite
nodes. run-unit-tests and run-unit-tests-locally run all unit tests in the
presently selected part of the Leo outline; run-all-unit-tests and
run-all-unit-tests-locally run all unit tests in the entire Leo outline.

Important: you must `run Leo in a console window`_ to see the output the
unit tests. Leo's unit test commands run all the unit tests using the
standard unittest text test runner, and the output of the unit tests
appears in the console.

test/unitTest.leo contains many examples of using @test and @suite nodes.

.. contents:: Contents
    :depth: 2
    :local:

#@+node:ekr.20210822060903.2: *5* Using @test nodes
**@test nodes** are nodes whose headlines start with @test. The unit test commands convert the body text of @test nodes into a unit test automatically. That is, Leo's unit test commands automatically create a unittest.TestCase instances which run the body text of the @test node. For example, let us consider one of Leo's actual unit tests. The headline is::

    @test consistency of back/next links

The body text is::

    if g.unitTesting:
        c,p = g.getTestVars() # Optional: prevents pychecker warnings.
        for p in c.all_positions():
            back = p.back()
            next = p.next()
            if back: assert(back.getNext() == p)
            if next: assert(next.getBack() == p)

When either of Leo's unit test commands finds this @test node the command will
run a unit test equivalent to the following::

    import leo.core.leoGlobals as g

    class aTestCase (unittest.TestCase):
        def shortDescription():
            return '@test consistency of back/next links'
        def runTest():
            c,p = g.getTestVars()
            for p in c.all_positions():
                back = p.back()
                next = p.next()
                if back: assert(back.getNext() == p)
                if next: assert(next.getBack() == p)

As you can see, using @test nodes saves a lot of typing:

- You don't have to define a subclass of unittest.TestCase.
- Within your unit test, the c, g and p variables are predefined, just like in Leo scripts.
- The entire headline of the @test node becomes the short description of the unit test.

**Important note**: notice that the first line of the body text is a **guard line**::

    if g.unitTesting:

This guard line is needed because this particular @test node is contained in the file leoNodes.py. @test nodes that appear outside of Python source files do not need guard lines. The guard line prevents the unit testing code from being executed when Python imports the leoNodes module; the g.unitTesting variable is True only while running unit tests.

**New in Leo 4.6**: When Leo runs unit tests, Leo predefines the 'self' variable to be the instance of the test itself, that is an instance of unittest.TestCase. This allows you to use methods such as self.assertTrue in @test and @suite nodes.

**Note**: Leo predefines the c, g, and p variables in @test and @suite nodes, just like in other scripts.  Thus, the line::

    c,p = g.getTestVars()

is not needed. However, it prevents pychecker warnings that c and p are undefined.
#@+node:ekr.20210822060903.3: *5* Using @suite nodes
**@suite nodes** are nodes whose headlines start with @suite. @suite nodes allow you to create and run custom subclasses of unittest.TestCase.

Leo's test commands assume that the body of an suite node is a script that creates a suite of tests and places that suite in g.app.scriptDict['suite']. Something like this::

    if g.unitTesting:
        __pychecker__ = '--no-reimport' # Prevents pychecker complaint.
        import unittest
        c,p = g.getTestVars() # Optional.
        suite = unittest.makeSuite(unittest.TestCase)
        << add one or more tests (instances of unittest.TestCase) to suite >>
        g.app.scriptDict['suite'] = suite

**Note**: as in @test nodes, the guard line, 'if unitTesting:', is needed only if the
@suite node appears in a Python source file.

Leo's test commands first execute the script and then run suite in g.app.scriptDict.get('suite') using the standard unittest text runner.

You can organize the script in an @suite nodes just as usual using @others, section references, etc. For example::

    if g.unitTesting:
        __pychecker__ = '--no-reimport'
        import unittest
        c,p = g.getTestVars() # Optional.
        # children define test1,test2..., subclasses of unittest.TestCase.
        @others 
        suite = unittest.makeSuite(unittest.TestCase)
        for test in (test1,test2,test3,test4):
            suite.addTest(test)
        g.app.scriptDict['suite'] = suite
#@+node:ekr.20210822060903.4: *5* Using @mark-for-unit-tests
When running unit tests externally, Leo copies any @mark-for-unit-tests nodes to dynamicUnitTest.leo.  Of course, this is in addition to all @test nodes and @suite nodes that are to be executed. You can use @mark-for-unit-test nodes to include any "supporting data" you want, including, say, "@common test code" to be imported as follows::

    exec(g.findTestScript(c,'@common test code'))

**Note**: putting @settings trees as descendants of an @mark-for-unit-test node will copy the @setting tree, but will *not* actually set the corresponding settings.
#@+node:ekr.20210822060903.5: *5* Test driven development in Leo
Test Driven Development (TDD) takes a bit of setup, but the initial investment repays itself many times over. To use TDD with Leo, start @test nodes with **preamble code**. As explained below, the preamble will do the following:

1. Optional: save the present outline if it has been changed.

2. Reload modules with imp.reload.

3. Create *new instances* of all objects under test.

Here is the actual preamble code used in Leo's import tests::

    if 0: # Preamble...
        # g.cls()
        if c.isChanged(): c.save()
        import leo.core.leoImport as leoImport
        import leo.plugins.importers.linescanner as linescanner
        import leo.plugins.importers.python
        import imp
        imp.reload(leo.plugins.importers.linescanner)
        imp.reload(leo.plugins.importers.python)
        imp.reload(leoImport)
        g.app.loadManager.createAllImporetersData()
        ic = leoImport.LeoImportCommands(c)
    else:
        ic = c.importCommands

    # run the test.
    ic.pythonUnitTest(p,s=s,showTree=True)
    
Let's look at this example in detail. These lines optionally clear the screen and save the outline::

    # g.cls()
    if c.isChanged(): c.save()

The next lines use imp.reload to re-import the affected modules::

    import leo.core.leoImport as leoImport
    import leo.plugins.importers.linescanner as linescanner
    import leo.plugins.importers.python
    import imp
    imp.reload(leo.plugins.importers.linescanner)
    imp.reload(leo.plugins.importers.python)
    imp.reload(leoImport)
    
Using imp.reload is usually not enough.  The preamble must *create new instances* of all objects under test. This can be a bit tricky. In the example above, the following lines create the new objects::

    g.app.loadManager.createAllImporetersData()
    ic = leoImport.LeoImportCommands(c)
    
The call to LM.createAllImporetersData() recomputes global tables describing importers. These tables must be updated to reflect possibly-changed importers. The call to leoImport.LeoImportCommands(c) creates a *new instance* of the c.importController. We want to use this new instance instead of the old instance, c.importController.

**Summary**
    
TDD makes a big difference when developing code. I can run tests repeatedly from the Leo outline that contains the code under test. TDD significantly improves my productivity.

Preamble code reload changed modules using imp.reload(). Preamble code must also create new instances of *all* objects that may have changed.

When creating several related unit tests, cutting and pasting the preamble from previous unit tests is usually good enough. @button scripts that create preamble code might be useful if you create lots of tests at once.
#@+node:ekr.20210822060903.6: *5* How the unit test commands work
The run-all-unit-tests-locally and run-unit-tests-locally commands run unit tests in the process that is running Leo. These commands *can* change the outline containing the unit tests.

The run-all-unit-tests and run-unit-tests commands run all tests in a separate process, so unit tests can never have any side effects. These commands never changes the outline from which the tests were run. These commands do the following:

1. Copy all @test, @suite, @unit-tests and @mark-for-unit-test nodes (including their descendants) to the file test/dynamicUnitTest.leo.

2. Run test/leoDynamicTest.py in a separate process.

   - leoDynamicTest.py opens dynamicUnitTest.leo with the leoBridge module.
     Thus, all unit tests get run with the nullGui in effect.

   - After opening dynamicUnitTest.leo, leoDynamicTest.py runs all unit tests
     by executing the leoTest.doTests function.

   - The leoTests.doTests function searches for @test and @suite nodes and
     processes them generally as described above. The details are a bit
     different from as described, but they usually don't matter. If you *really*
     care, see the source code for leoTests.doTests.
#@+node:ekr.20210822060903.7: *5* \@button timer
The timit button in unitTest.leo allows you to apply Python's timeit module. See http://docs.python.org/lib/module-timeit.html. The contents of @button timer is::

    import leo.core.leoTest as leoTest
    leoTest.runTimerOnNode(c,p,count=100)

runTimerOnNode executes the script in the presently selected node using timit.Timer and prints the results.
#@+node:ekr.20210822060903.8: *5* \@button profile
The profile button in unitTest.leo allows you to profile nodes using Python's profiler module. See http://docs.python.org/lib/module-profile.html The contents of @button profile is::

    import leo.core.leoTest as leoTest
    leoTest.runProfileOnNode(p,outputPath=None) # Defaults to leo\test\profileStats.txt

runProfileOnNode runs the Python profiler on the script in the selected node, then reports the stats.
#@+node:ekr.20210822061141.1: *4* Tip: run unit tests locally using clones
Running a unit test locally, without exiting Leo, saves a lots of time.  It's much faster than having to load unitTest.leo or even a small .leo file.

The question is, how to use the newest code?  imp.reload often doesn't work. But there is a trick that does work.  Clone the code under development and put it under an @test node.  The script in the @test node uses @others to gain access to the code, not an import.

For instance, here is the @test node I use to develop the new javascript importer::

    g.cls()
    p1 = p.copy()
    if c.isChanged():
        c.save()
    import leo.plugins.importers.basescanner as basescanner
    @others
    scanner = JavaScriptScanner(c.importCommands)
    h = '@ignore js-test'
    p = g.findNodeAnywhere(c, h)
    if p:
        while p.firstChild():
            p.firstChild().doDelete()
    else:
        p = c.insertHeadline()
        p.h = h
    c.selectPosition(p)
    fn = r'c:\prog\jQuery-short2.js'
    s = open(fn, 'r').read()
    print('Sources..\n\n%s\n\n' % s)
    scanner.scan(s, p)
    c.selectPosition(p1)
    c.redraw()
    print('done')

To repeat, the code under test is a child of this node, so the script uses @others to gain access to it.  It's super fast.
#@+node:ekr.20210822061214.1: *3* LeoU: Unit tests should not depend on settings
.. https://github.com/leo-editor/leo-editor/issues/527

Relying on settings nodes in unitTest.leo risks creating hard-to-find dependencies between seemingly unrelated unit tests.  Instead, unit tests can follow the pattern for, say, `@test add python comments`::


    w = c.frame.body.wrapper
    p = g.findNodeInTree(c,p,'python')
    assert p,'not found: python'
    # Save the initial value of the setting.
    old_indent = c.config.getBool('indent_added_comments',default=True)
    table = (
        (
            True,
            '@language python\ndef spam():\n    pass\n\n# after',
            '@language python\ndef spam():\n    # pass\n\n# after',
        ),
        (
            False,
            '@language python\ndef spam():\n    pass\n\n# after',
            '@language python\ndef spam():\n#     pass\n\n# after',
        ),
    )
    try:
        for indent, s1, expected in table:
            # Step 1: set the setting.
            c.config.set(None, 'bool', 'indent_added_comments', indent, warn=False)
            val = c.config.getBool('indent_added_comments')
            assert indent == val, (repr(indent), repr(val))
            # Step 2: set p.b and the insert point.
            c.selectPosition(p)
            p.b = s1
            i = p.b.find('pass')
            assert i > -1,'fail1: %s' % (repr(p.b))
            w.setSelectionRange(i,i+4)
            # Step 3: test add-comments
            c.addComments()
            assert p.b == expected, ('indent: %5s got:\n%r\nexpected:\n%r' % (indent, p.b, expected))
    finally:
        # Restore the initial value of the setting.
        c.config.set(p, 'bool', 'indent_added_comments', old_indent)
        val = c.config.getBool('indent_added_comments')

This code saves and restores the actual value of the `@bool indent_added_comments` setting. It uses::

    c.config.set(c.config.set(None, 'bool', 'indent_added_comments', indent, warn=False)

to set the setting to the value of 'indent'.  Note that c.config.set is seldom (never?) used in Leo's core.
#@+node:ekr.20210822061319.1: *3* How can I use dev nodes to develop and test Leo's own code?
Here are step-by-step instructions for developing code in Leo without having to reload Leo.

1. Create a **dev node** as a test harness.

This node (in the Leo outline) defines your development environment. You can use an @test node, an @button node, or an @command node. Using a plain outline node would be less convenient because you won't be able to execute it so easily.

The **dev script** is the script in the dev node itself.  The **code under development** is the code exercised by the dev script.

This is the most important step! Once you say to yourself, "Ok, I'm going to be clever and develop my code using a dev script, everything else will happen naturally. That is, the programming process itself will lead you to the next action. If you don't create a custom dev environment you will have to reload Leo to test your new code. That will be much slower.

2. Set up your dev node so you can execute it with a single keystroke.

You can do any of the following:

- Use a marked @test node and use run-marked-unit-tests-locally.
- Disable or ignore all other @test nodes and then use run-all-unit-tests-locally.
- Bind a keystroke to an @button or @command node.
- Run an @button or @command node the first time from the minibuffer, and then re-execute it with Ctrl-P (repeat-complex-command.

3. Define or access the code under development.

There are two main ways to do this:

A. Clone the code under test and move the clones so they are children of the dev node. Access the code under test using @others in the top-level node.

B. Import one or more modules containing the code under test. Access the code using imp.reload to ensure that all imported modules are up-to-date.

4. Create new instances of objects

**You must not use existing Leo objects when testing Leo code.**  This includes all objects accessible via c, g and p, either directly or indirectly.  For example, you must not use c, c.frame, c.frame.body, etc., etc., even if you have reloaded all of Leo's modules!

You will seldom need to worry about reloading code if you use @others to define the code under test.

5. Run the test, edit, repeat

After creating the dev node you simply run the dev script until everything works :-)  The details depend on the code being developed.  Otoh, we can safely assume that devs can handle problems as they arise.

Summary

@test, @command or @button can be thought of as defining an (almost) pristine dev environment. This is another way of describing the Stupendous Aha. The initial cost of creating dev nodes pays off immediately.

Dev scripts should create a new, pristine environment every time it they are executed, using imp.reload as needed. Dev scripts should always create new objects for testing.  Dev scripts may use Leo's core objects provided they have not been modified.

Dev scripts have many advantages.  They can:

- access code using clones.
- use outlines to organize the code under development.
- form a permanent record of completed work.
- can morph into unit tests.
#@+node:ekr.20210822061354.1: *3* How do I run unit tests from Leo?
Leo makes it easy to create and run unit tests from individual outline nodes or trees. A node whose headline starts with @test defines a unit test. The body text of the @test node contains a **self-contained** unit test. For example, this creates a complete unit test::

    @test fails  (headline)
    assert False (body text)

To run this test, select the @test node and do::

    <alt-x>run-selected-unit-tests-locally.

Leo will create and run the unit test automatically.

To see all of Leo's unit testing commands, do::

    <alt-x>run<tab>

Leo pre-defines 'c', 'g' and 'p' in unit tests just as in scripts.

.. _`unit testing`: unitTesting.html

For more details about unit testing, the `unit testing`_ page.

**Notes for Leo developers**

leo/test/unitTest.leo contains all of Leo's own unit tests.

Running all tests is not necessary.  Just select::

    Active Unit Tests

and then do Alt-4 (run-selected-unit-tests-locally).

**Note**: Some tests will likely fail on machines other than EKR's. You only need to be concerned about unit tests that start failing after you make your changes.
#@+node:ekr.20210815060206.1: ** From leoDist.leo

#@+node:ekr.20210815060352.1: *3* --- from setup.py
@language python
#@+node:ekr.20210815060206.2: *4* get_version (not used)
def get_version(file, version=None):
    """Determine current Leo version. Use git if in checkout, else internal Leo"""
    root = os.path.dirname(os.path.realpath(file))
    if os.path.exists(os.path.join(root, '.git')):
        version = git_version(file)
    if not version:
        version = get_semantic_version(leoVersion.version)
    return version
#@+node:ekr.20210815060206.3: *4* git_version (not used)
def git_version(file, version=None):
    """
    Fetch from Git: {tag} {distance-from-tag} {current commit hash}
    Return as semantic version string compliant with PEP440
    """
    root = os.path.dirname(os.path.realpath(file))
    try:
        tag, distance, commit = g.gitDescribe(root)  # 5.6b2, 55, e1129da
        ctag = clean_git_tag(tag)
        #version = get_semver(ctag)
        version = ctag
        if int(distance) > 0:
            version = '{}-dev{}'.format(version, distance)
    except IndexError:
        print('Attempt to `git describe` failed with IndexError')
    except FileNotFoundError:
        print('Attempt to `git describe` failed with FileNotFoundError')
    return version
#@+node:ekr.20210815060206.4: *5* clean_git_tag
def clean_git_tag(tag):
    """Return only version number from tag name. Ignore unknown formats.
       Is specific to tags in Leo's repository.
            5.7b1          -->	5.7b1
            Leo-4-4-8-b1   -->	4-4-8-b1
            v5.3           -->	5.3
            Fixed-bug-149  -->  Fixed-bug-149
    """
    if tag.lower().startswith('leo-'): tag = tag[4:]
    if tag.lower().startswith('v'): tag = tag[1:]
    return tag
#@+node:ekr.20210815060206.5: *4* get_semantic_version (not used)
def get_semantic_version(tag):
    """Return 'Semantic Version' from tag string"""
    try:
        import semantic_version
    except Exception:
        print(f"Can not import semantic_version. Using {tag!r}")
        return tag
    try:
        # tuple of major, minor, build, pre-release, patch.
        # Example: 5.6b2 --> 5.6-b2
        return str(semantic_version.Version.coerce(tag, partial=True))
    except Exception:
        ### g.es_exception()
        print_exception()
        print(f"Bad version: {tag!r}")
        return tag
        ###
        # print(
            # f"*** Failed to parse Semantic Version from git tag '{tag}'.\n"
            # "Expecting tag name like '5.7b2', 'leo-4.9.12', 'v4.3' for releases.\n"
            # "This version can't be uploaded to PyPi.org.")
        # version = tag
    # return version
#@+node:ekr.20210815060405.1: *3* --- old distribution files
@language rest
@wrap

-matt (@maphew)

Old distribution files.
#@+node:ekr.20210815060405.2: *4* @@clean ../../About leo.exe.TXT
leo.exe contains everything you need to run Leo on Windows without
installing *anything* else. It contains all needed libraries, including
Python itself, PyQt, pylint and other packages. It also contains many data
files, including Leo's source code and various .leo files.

Notes:

1. leo.exe was created by PyInstaller: http://pythonhosted.org/PyInstaller

2. Before Leo starts, PyInstaller unpacks all libraries and data files to a
   temp folder: ~\AppData\Local\Temp\_MEInnn. As a result, starting leo.exe
   is slower than usual.

3. *Warning*: You can open Leo files from the "Files:Open Leo File" menu,
   but these files will be in the temp folder. They will be *destroyed*
   when you exit leo.exe. If you want to do real work with Leo you must
   create .leo files in a permanent folder on your machine.
#@+node:ekr.20210815060405.3: *4* @@clean ../../optional-tools.txt
# A list of pip installable tools and libraries in the Requirements File Format
# https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
#
# Install with:
#   
#       pip install -r optional-tools.txt
#

pyenchant  # spell check support. No wheels for some platforms, e.g. amd64
pyxml; python_version < "3.0"   # Used for XML import. Appears to be only python 2 package
#@+node:ekr.20210815060405.4: *4* @@clean ../../INSTALL.TXT
System requirements
-------------------

Leo requires the `Python`_ and `PyQt_` package.
The `PyEnchant`_ package is optional. 

**Python**: Leo will work on any platform that supports Python 2.6 or
above, including Python 3.0 and above. To install Python, see
http://python.org.

**PyQt**: PyQt provides Leo's widgets. To install PyQt, get the binary
package from: http://www.riverbankcomputing.co.uk/software/pyqt/download
The PyQt version must match your installed Python version. Remember that
Leo requires Python 2.6 or later, or Python 3.0 or later. Now run the
binary PyQt installer.

**PyEnchant**: You must install the PyEnchant package if you want to use
Leo's Spell tab. Download and install the PyEnchant package from
http://pythonhosted.org/pyenchant/download.html There is an executable
installer for Windows users.


Leo's HOME directory
--------------------

Python's HOME environment variable specifies Leo's HOME directory.
See http://docs.python.org/lib/os-procinfo.html for details.

Leo puts several files in your HOME/.leo directory:
.leoID.txt, .leoRecentFiles.txt, and myLeoSettings.leo.
There are various fallback directories if there is no home directory.

Installing Leo on Linux
-----------------------

You may download Leo's sources in one of three ways, as described at:
http://leoeditor.com/download.html If the sources are zipped, unzip them
into a folder in your home directory, say ~/leo-5.2 (or /usr/bin, etc.)

To gain access to Leo, you have two choices:

A. Add  ~/leo-5.2 to your path.
B. Create alias for Leo.

I prefer using aliases because they allow me to start Leo using Python 2 or 3,
or other distributions, like Anaconda 2 or 3.

See `Running Leo`_ for how to run Leo after installing it.

Installing Leo on Windows
--------------------------

Install Python and Qt, as described above.

You may download Leo's sources in one of three ways, as described at:
http://leoeditor.com/download.html If the sources are zipped, unpack them
into a temp folder. You may place the sources anywhere you like, including
Python's *site-packages* folder, for example, C:\Python36\Lib\site-packages.


Running Leo
-----------

You can run Leo from a Python interpreter as follows::

    import leo
    leo.run() # runs Leo, opening a new outline or,
    leo.run(fileName=aFileName) # runs Leo, opening the given file name.

Another way to run Leo is as follows::

    cd <path-to-launchLeo.py>
    python launchLeo.py

Creating File Associations
--------------------------

**Linux**

The following shell script will allow you to open foo.leo files by typing leo foo::

    #!/bin/sh 
    python <leopath>launchLeo.py $1

where <leopath> is the path to the directory *containing* the leo directory. 

**Windows**

*Important*: Leo's binary Windows installer sets file associations
automatically, so this section is needed only if you are installing Leo
from a .zip file or other sources.

There are two ways of associating .leo files with Leo. The first uses the
Windows control panel, the second, the Windows console.

**Method 1: Using the Windows Control Panel**

The goal is that you want to associate .leo files with the following command::

    "<path to python>\python.exe" "<path to launchLeo.py>\launchLeo.py" "%1"
    
Before Windows 7, you do this with using the Folder Options control panel.
In Windows 7, you do this with the Default Programs control panel.

*Note*: "%1" passes just the file being clicked on, quoted for spaces etc.
The quotation marks are needed to handle file paths containing spaces.

*Warning:* In a batch file, %1 passes just the first command line parameter.
It is logical to expect %* to work for file associations just as in batch
files. Alas, it does not.

**Method 2: Using the Windows Console**

Open a Windows console with administrator privileges, then type::

    ftype LeoFile="<path to python>\pythonw.exe" "<path to launchLeo.py>\launchLeo.py" "%1" %*
    assoc .leo=LeoFile

And put this leo.bat in %PATH%::

    @start /b "Leo" "<path to python>\python.exe" "<path to launchLeo.py>\launchLeo.py" %*
    
You may omit the /b option if you want to create a separate console window for Leo.

Adding Leo to Your Path
-----------------------

After you have installed Leo, you should add the location of your leo/core folder to your python path.
One way to do this is adding something like the following to python/Lib/sitecustomize.py:

    import sys
    sys.path.append(r'<path-to-leo>leo\core')

Another way is to append <path-to-leo> to the Windows PYTHONPATH environment variable.

Running Leo for the first time
------------------------------

The first time you start Leo, a dialog will ask you for a unique identifier. If
you are using cvs, use your cvs login name. Otherwise your initials will do. Leo
stores this identifier in the file ``.leoID.txt``. Leo attempts to create
``leoID.txt`` in the .leo sub-directory of your home directory, then in Leo's config directory, and
finally in Leo's core directory. You can change this identifier at any time by
editing ``.leoID.txt``.
#@+node:ekr.20210312150753.1: ** leoTangle.py
'''
Support for @root and Leo's tangle and untangle commands.
'''
# Removed from Leo's code base 2021/03/08.
import os
import leo.core.leoGlobals as g
<< about Tangle and Untangle >>
<< constants >>
@others
@language python
@tabwidth -4
@pagewidth 70
#@+node:ekr.20210312150753.2: *3* << About Tangle and Untangle >>
@root directive. The stack never becomes empty because of the entry
@
The Tangle command translates the selected @root tree into one or more
well-formatted source files. The outline should contain directives,
sections references and section definitions, as described in Chapter
4. The Untangle command is essentially the reverse of the Tangle
command. The Tangle command creates a derived file from an @root tree;
the Untangle command incorporates changes made to derived files back
into the @root tree.

The Tangle command operates in two passes. The first pass discovers
the complete definitions of all sections and places these definitions
in a Tangle Symbol Table. The first pass also makes a list of root
sections. Definitions can appear in any order, so we must scan the
entire input file to know whether any particular definition has been
completed.

Tangle's second pass creates one file for each @root node. Tangle
rescans each section in the list of roots, copying the root text to
the output and replacing each section reference by the section's
definition. This is a recursive process because any definition may
contain other references. We can not allow a section to be defined in
terms of itself, either directly or indirectly. We check for such
illegally recursive definitions in pass 2 using the section stack
class. Tangle indicates where sections begin and end using comment
lines called sentinel lines. The sentinels used predate the formats
described in the "Format of external files" appendix.

The key design principle of the Tangle command is this: Tangle must
output newlines in a context-free manner. That is, Tangle must never
output conditional newlines, either directly or indirectly. Without
this rule Untangle could not determine whether to skip or copy
newlines.

The Tangle command increases the indentation level of a section
expansion the minimum necessary to align the section expansion with
the surrounding code. In essence, this scheme aligns all section
expansions with the line of code in which the reference to the section
occurs. In some cases, several nested sections expansions will have
the same indentation level. This can occur, for example, when a
section reference in an outline occurs at the left margin of the
outline.

This scheme is probably better than more obvious schemes that indent
more "consistently." Such schemes would produce too much indentation
for deeply nested outlines. The present scheme is clear enough and
avoids indentation wherever possible, yet indents sections adequately.
End sentinel lines make this scheme work by making clear where the
expansion of one section ends and the expansion of a containing
section resumes.

Tangle increases indentation if the section reference does not start a
line. Untangle is aware of this hack and adjusts accordingly. This
extra indentation handles several common code idioms, which otherwise
would create under-indented code. In short, Tangle produces highly
readable, given the necessity of preserving newlines for Untangle.

Untangle is inherently complex. It must do a perfect job of updating
the outline, especially whitespace, from expansions of section
definitions created by the Tangle command. Such expansions need not be
identical because they may have been generated at different levels of
indentation. The Untangle command can not assume that all expansions
of a section will be identical in the derived file; within the derived
file, the programmer may have made incompatible changes to two
different expansions of the same section. Untangle must check to see
that all expansions of a section are "equivalent". As an added
complication, derived files do not contain all the information found
in @root trees. @root trees may contain headlines that generate no
code at all. Also, an outline may define a section in several ways:
with an @c or @code directive or with a section definition line. To be
useful, Untangle must handle all these complications flawlessly.

Untangle operates in three passes. The first pass builds a symbol
table in the same way that Tangle does. The key information there
informs how the second pass finds definitions in the derived file: to
support multi-language files (e.g., javascript embedded in html) the
second pass needs to know what comment delimiters to look for as it
discovers sentinels in the derived file. Using comment delimiters as
suggested by the first pass, it uses the sentinels to find section
parts and enters them into the Untangle Symbol Table, or UST.
Definitions often include references to other sections, so definitions
often include nested definitions of referenced sections. The second
pass of Untangle uses a definition stack to keep track of nested
definitions. The top of the stack represents the definition following
the latest reference, except for the very first entry pushed on the
stack, which represents the code in the outline that contains the
for the @root section. All definitions of a section should
match--otherwise there is an inconsistent definition. This pass uses a
forgiving compare routine that ignores differences that do not affect
the meaning of a program.

Untangle's third pass enters definitions from the outline into a
second Tangle Symbol Table, or TST. The third pass simultaneously
updates all sections in the outline whose definition in the new TST
does not match the definition in the UST. The central coding insight
of the Untangle command is that the second pass of Untangle is almost
identical to the first pass of Tangle! That is, Tangle and Untangle
share key parts of code, namely the skip_body() method and its allies.
Just when skip_body() enters a definition into the symbol table, all
the information is present that Untangle needs to update that
definition.
#@+node:ekr.20210312150753.3: *3* << constants >>
max_errors = 20
# All these must be defined together, because they form a single enumeration.
# Some of these are used by utility functions.
# Used by token_type().
plain_line = 1 # all other lines
at_at = 2 # double-at sign.
at_chapter = 3 # @chapter
# at_c     = 4 # @c in noweb mode
at_code = 5 # @code, or @c or @p in CWEB mode.
at_doc = 6 # @doc
at_other = 7 # all other @directives
at_root = 8 # @root or noweb * sections
at_section = 9 # @section
# at_space = 10 # @space
at_web = 11 # any CWEB control code, except at_at.
# Returned by self.skip_section_name() and allies and used by token_type.
bad_section_name = 12 # < < with no matching > >
section_ref = 13 # < < name > >
section_def = 14 # < < name > > =
# Returned by is_sentinal_line.
non_sentinel_line = 15
start_sentinel_line = 16
end_sentinel_line = 17
# Stephen P. Schaefer 9/13/2002
# add support for @first
at_last = 18
#@+node:ekr.20210312150753.4: *3* node classes
#@+node:ekr.20210312150753.5: *4* class TstNode
class TstNode:
    @others
#@+node:ekr.20210312150753.6: *5* TstNode.__init__
def __init__(self, name, root_flag):

    self.name = name
    self.is_root = root_flag
    self.referenced = False
    self.parts = []
    self.delims = None
#@+node:ekr.20210312150753.7: *5* TstNode.__repr__
def __repr__(self):
    return "TstNode:" + self.name
#@+node:ekr.20210312150753.8: *5* TstNode.dump
def dump(self):
    s = ("\nsection: " + self.name +
        ", referenced:" + str(self.referenced) +
        ", is root:" + str(self.is_root))
    if self.parts:
        s += "\n----- parts of " + g.angleBrackets(self.name)
        n = 1 # part list is in numeric order
        for part in self.parts:
            s += "\n----- Part " + str(n)
            n += 1
            s += "\ndoc:  [" + repr(part.doc) + "]"
            s += "\ncode: [" + repr(part.code) + "]"
            s += "\ndelims: <%s><%s><%s>" % part.delims
            for ref in part.reflist():
                s += "\n    ref: [" + repr(ref.name) + "]"
        s += "\n----- end of partList\n"
    return s
#@+node:ekr.20210312150753.9: *4* class PartNode
class PartNode:
    @others
#@+node:ekr.20210312150753.10: *5* PartNode.__init__
def __init__(self, name, code, doc, is_root, is_dirty, delims):

    self.name = name # Section or file name.
    self.code = code # The code text.
    self.doc = doc # The doc text.
    self.is_dirty = is_dirty # True: VNode for body text is dirty.
    self.is_root = is_root # True: name is a root name.
    self.delims = delims
    self.refs = []
#@+node:ekr.20210312150753.11: *5* PartNode.__repr__
def __repr__(self):
    return "PartNode:" + self.name
#@+node:ekr.20210312150753.12: *5* PartNode.reflist
def reflist(self, refs=False):
    if refs:
        self.refs = refs
    return self.refs
#@+node:ekr.20210312150753.13: *4* class UstNode
class UstNode:
    @others
#@+node:ekr.20210312150753.14: *5* UstNode.__init__
@
The text has been masssaged so that 1) it contains no leading
indentation and 2) all code arising from section references have been
replaced by the reference line itself. Text for all copies of the same
part can differ only in non-critical white space.
@c

def __init__(self, name, code, part, of, nl_flag, update_flag):

    self.name = name # section name
    self.parts = {} # parts dict
    self.code = code # code text
    self.part = part # n in "(part n of m)" or zero.
    self.of = of # m in "(part n of m)" or zero.
    self.nl_flag = nl_flag # True: section starts with a newline.
    self.update_flag = update_flag # True: section corresponds to a section in the outline.
#@+node:ekr.20210312150753.15: *5* UstNode.__repr__
def __repr__(self):
    return "UstNode:" + self.name
#@+node:ekr.20210312150753.16: *5* UstNode.dump
def dump(self):
    s = "name: %s" % repr(self.name)
    for part in self.parts.values():
        s += "\n----- part %s of %s -----\n" % (repr(part.part), repr(part.of))
        s += repr(g.get_line(part.code, 0))
        s += "\nupdate_flag: %s" % repr(part.update_flag)
    return s
#@+node:ekr.20210312150753.17: *4* class DefNode
class DefNode:
    @others
#@+node:ekr.20210312150753.18: *5* DefNode.__init__
@
The text has been masssaged so that 1) it contains no leading
indentation and 2) all code arising from section references have been
replaced by the reference line itself. Text for all copies of the same
part can differ only in non-critical white space.
@c

def __init__(self, name, indent, part, of, nl_flag, code):

    self.name = name
    self.indent = indent
    self.code = code
    if self.code is None: self.code = ""
    self.part = part
    self.of = of
    self.nl_flag = nl_flag
#@+node:ekr.20210312150753.19: *5* DefNode.__repr__
def __repr__(self):
    return "DefNode:" + self.name
#@+node:ekr.20210312150753.20: *4* class RootAttributes (Stephen P. Schaefer)
@ Stephen P. Schaefer, 9/2/2002
Collect the root node specific attributes in an
easy-to-use container.
@c

class RootAttributes:
    @others
#@+node:ekr.20210312150753.21: *5* RootAttributes.__init__
@ Stephen P. Schaefer, 9/2/2002
Keep track of the attributes of a root node
@c

def __init__(self, tangle_state):

    self.language = tangle_state.language
    self.use_header_flag = tangle_state.use_header_flag
    self.print_mode = tangle_state.print_mode
    # of all the state variables, this one isn't set in TangleCommands.__init__
    # peculiar
    try:
        self.path = tangle_state.path
    except AttributeError:
        self.path = None
    self.page_width = tangle_state.page_width
    self.tab_width = tangle_state.tab_width
    self.first_lines = tangle_state.first_lines # Stephen P. Schaefer 9/13/2002
#@+node:ekr.20210312150753.22: *5* RootAttributes.__repr__
def __repr__(self):
    return ("RootAttributes: language: " + self.language +
        ", use_header_flag: " + repr(self.use_header_flag) +
        ", print_mode: " + self.print_mode +
        ", path: " + self.path +
        ", page_width: " + repr(self.page_width) +
        ", tab_width: " + repr(self.tab_width) +
        # Stephen P. Schaefer 9/13/2002
        ", first_lines: " + self.first_lines)
#@+node:ekr.20210312150753.23: *3* class TangleCommands
class BaseTangleCommands:
    """The base class for Leo's tangle and untangle commands."""
    @others

class TangleCommands(BaseTangleCommands):
    """A class that implements Leo' tangle and untangle commands."""
    pass
#@+node:ekr.20210312150753.24: *4* class RegexpForLanguageOrComment
class RegexpForLanguageOrComment:
    import re
    regex = re.compile(r'''
        ^(
            (?P<language>
                @language\s[^\n]*
            ) |
            (?P<comment>
                @comment\s[^\n]*
            ) |
            (
                [^\n]*\n
            )
        )*'''         , re.VERBOSE)
#@+node:ekr.20210312150753.25: *4* tangle.Birth
def __init__(self, c):
    self.c = c
    self.init_ivars()

def init_ivars(self):
    c = self.c
    g.app.scanErrors = 0
    << init tangle ivars >>
    << init untangle ivars >>

# Called by scanAllDirectives

def init_directive_ivars(self):
    c = self.c
    << init directive ivars >>
    
def reload_settings(self):
    c = self.c
    self.output_doc_flag = c.config.getBool('output-doc-flag')
    self.tangle_batch_flag = c.config.getBool('tangle-batch-flag')
    self.untangle_batch_flag = c.config.getBool('untangle-batch-flag')
    self.use_header_flag = c.config.getBool('use-header-flag')
    
reloadSettings = reload_settings
#@+node:ekr.20210312150753.26: *5* << init tangle ivars >>
# Various flags and counts...
self.errors = 0 # The number of errors seen.
# self.tangling = True # True if tangling, False if untangling.
self.path_warning_given = False # True: suppress duplicate warnings.
self.tangle_indent = 0 # Level of indentation during pass 2, in spaces.
if c.frame:
    self.file_name = c.mFileName # The file name (was a bridge function)
else:
    self.file_name = "<unknown file name>"
self.p = None # position being processed.
self.output_file = None # The file descriptor of the output file.
self.start_mode = "doc" # "code" or "doc".  Use "doc" for compatibility.
self.tangle_output = {} # For unit testing.
@ Symbol tables: the TST (Tangle Symbol Table) contains all section names in the outline.
The UST (Untangle Symbol Table) contains all sections defined in the derived file.
@c
self.tst = {}
self.ust = {}
# The section stack for Tangle and the definition stack for Untangle.
self.section_stack = []
self.def_stack = []
@
The list of all roots. The symbol table routines add roots to self
list during pass 1. Pass 2 uses self list to generate code for all
roots.
@c
self.root_list = []
# The filename following @root in a headline.
# The code that checks for < < * > > = uses these globals.
self.root = None
self.root_name = None
# Formerly the "tangle private globals"
# These save state during tangling and untangling.
# It is possible that these will be removed...
if 1:
    self.head_root = None
    self.code = None
    self.doc = None
    self.header_name = None
    self.header = None
    self.section_name = None
@
The following records whether we have seen an @code directive in a
body text. If so, an @code represents < < header name > > = and it is
valid to continue a section definition.
@c
self.code_seen = False # True if @code seen in body text.
# Support of output_newline option
self.output_newline = g.getOutputNewline(c=c)
#@+node:ekr.20210312150753.27: *5* << init untangle ivars >>
@ Untangle vars used while comparing.
@c
self.line_comment = self.comment = self.comment_end = None
self.comment2 = self.comment2_end = None
self.string1 = self.string2 = self.verbatim = None
self.message = None # forgiving compare message.
#@+node:ekr.20210312150753.28: *5* << init directive ivars >> (tangle)
# Global options
self.page_width = c.page_width
self.tab_width = c.tab_width
# Get settings from reload_settings.
self.output_doc_flag = False
self.tangle_batch_flag = False
self.untangle_batch_flag = False
self.use_header_flag = False
# Default tangle options.
self.tangle_directory = None # Initialized by scanAllDirectives
# Default tangle language
if c.target_language: c.target_language = c.target_language.lower()
self.language = c.target_language
if 0: # debug
    import sys
    g.es(f"TangleCommands.languague: {self.language} at header {self.p!r}")
    f = sys._getframe(1)
    g.es("caller: " + f.f_code.co_name)
    f = sys._getframe(2)
    g.es("caller's caller: " + f.f_code.co_name)
# Abbreviations for self.language.
# Warning: these must also be initialized in tangle.scanAllDirectives.
self.raw_cweb_flag = (self.language == "cweb") # A new ivar.
# Set only from directives.
self.print_mode = "verbose"
self.first_lines = ""
self.encoding = c.config.default_derived_file_encoding # 2/21/03
self.output_newline = g.getOutputNewline(c=c) # 4/24/03: initialize from config settings.
#@+node:ekr.20210312150753.29: *4* top level
@ Only top-level drivers initialize ivars.
#@+node:ekr.20210312150753.30: *5* cleanup
# This code is called from tangleTree and untangleTree.

def cleanup(self):
    # Reinitialize the symbol tables and lists.
    self.tst = {}
    self.ust = {}
    self.root_list = []
    self.def_stack = []
#@+node:ekr.20210312150753.31: *5* initTangleCommand
def initTangleCommand(self):
    c = self.c
    c.endEditing()
    if not g.unitTesting:
        g.es("tangling...")
    self.init_ivars()
    self.tangling = True
#@+node:ekr.20210312150753.32: *5* initUntangleCommand
def initUntangleCommand(self):
    c = self.c
    c.endEditing()
    if not g.unitTesting:
        g.es("untangling...")
    self.init_ivars()
#@+node:ekr.20210312150753.33: *5* tangle
def tangle(self, event=None, p=None):
    c = self.c
    if not p: p = c.p
    self.initTangleCommand()
    # Paul Paterson's patch.
    if not self.tangleTree(p, report_errors=True):
        g.es("looking for a parent to tangle...")
        while p:
            assert(self.head_root is None)
            d = g.get_directives_dict(p, [self.head_root])
            if 'root' in d:
                g.es("tangling parent")
                self.tangleTree(p, report_errors=True)
                break
            p.moveToParent()
    if not g.unitTesting:
        g.es("tangle complete")
#@+node:ekr.20210312150753.34: *5* tangleAll
def tangleAll(self, event=None):
    c = self.c
    self.initTangleCommand()
    has_roots = False
    for p in c.rootPosition().self_and_siblings():
        ok = self.tangleTree(p, report_errors=False)
        if ok: has_roots = True
        if self.path_warning_given:
            break # Fatal error.
    self.errors += g.app.scanErrors
    if not has_roots:
        self.warning("----- the outline contains no roots")
    elif self.errors > 0 and not self.path_warning_given:
        self.warning("----- tangle halted because of errors")
    else:
        if not g.unitTesting:
            g.es("tangle complete")
#@+node:ekr.20210312150753.35: *5* tangleMarked
def tangleMarked(self, event=None):
    c = self.c; p = c.rootPosition()
    c.clearAllVisited() # No roots have been tangled yet.
    self.initTangleCommand()
    any_marked = False
    while p:
        is_ignore, i = g.is_special(p.b, "@ignore")
        # Only tangle marked and unvisited nodes.
        if is_ignore:
            p.moveToNodeAfterTree()
        elif p.isMarked():
            ok = self.tangleTree(p, report_errors=False)
            if ok: any_marked = True
            if self.path_warning_given:
                break # Fatal error.
            p.moveToNodeAfterTree()
        else: p.moveToThreadNext()
    self.errors += g.app.scanErrors
    if not any_marked:
        self.warning("----- The outline contains no marked roots")
    elif self.errors > 0 and not self.path_warning_given:
        self.warning("----- Tangle halted because of errors")
    else:
        if not g.unitTesting:
            g.es("tangle complete")
#@+node:ekr.20210312150753.36: *5* tanglePass1
# Traverses the tree whose root is given, handling each headline and associated body text.

def tanglePass1(self, p_in, delims):
    """The main routine of tangle pass 1"""
    p = self.p = p_in.copy() # self.p used by update_def in untangle
    self.setRootFromHeadline(p)
    theDict = g.get_directives_dict(p, [self.head_root])
    if ('ignore' in theDict):
        return
    self.scanAllDirectives(p) # calls init_directive_ivars.
    # Scan the headline and body text.
    # @language and @comment are not recognized in headlines
    self.skip_headline(p)
    delims = self.skip_body(p, delims)
    if self.errors + g.app.scanErrors >= max_errors:
        self.warning("----- Halting Tangle: too many errors")
    elif p.hasChildren():
        p.moveToFirstChild()
        self.tanglePass1(p, delims)
        while p.hasNext() and (self.errors + g.app.scanErrors < max_errors):
            p.moveToNext()
            self.tanglePass1(p, delims)
#@+node:ekr.20210312150753.37: *5* tanglePass2
# At this point p is the root of the tree that has been tangled.

def tanglePass2(self):
    self.p = None # self.p is not valid in pass 2.
    self.errors += g.app.scanErrors
    if self.errors > 0:
        self.warning("----- No file written because of errors")
    elif self.root_list is None:
        self.warning("----- The outline contains no roots")
    else:
        self.put_all_roots() # pass 2 top level function.
#@+node:ekr.20210312150753.38: *5* tangleTree (calls cleanup)
# This function is called only from the top level,
# so there is no need to initialize globals.

def tangleTree(self, p, report_errors):
    """Tangles all nodes in the tree whose root is p.

    Reports on its results if report_errors is True."""
    p = p.copy() # 9/14/04
    assert(p)
    any_root_flag = False
    next = p.nodeAfterTree()
    self.path_warning_given = False
    self.tangling = True
    while p and p != next:
        self.setRootFromHeadline(p)
        assert self.head_root is None
        theDict = g.get_directives_dict(p, [self.head_root])
        is_ignore = 'ignore' in theDict
        is_root = 'root' in theDict
        is_unit = 'unit' in theDict
        if is_ignore:
            p.moveToNodeAfterTree()
        elif not is_root and not is_unit:
            p.moveToThreadNext()
        else:
            self.scanAllDirectives(p) # sets self.init_delims
            self.tanglePass1(p, self.init_delims) # sets self.p
            if self.root_list and self.tangling:
                any_root_flag = True
                self.tanglePass2() # self.p invalid in pass 2.
            self.cleanup()
            p.moveToNodeAfterTree()
            if self.path_warning_given: break # Fatal error.
    if self.tangling and report_errors and not any_root_flag:
        # This is done by Untangle if we are untangling.
        self.warning("----- The outline contains no roots")
    return any_root_flag
#@+node:ekr.20210312150753.39: *5* untangle
def untangle(self, event=None, p=None):
    c = self.c
    if not p:
        p = c.p
    self.initUntangleCommand()
    # must be done at this point, since initUntangleCommand blows away tangle_output
    << read fake files for unit testing >>
    self.untangleTree(p, report_errors=True)
    if not g.unitTesting:
        g.es("untangle complete")
    c.redraw()
#@+node:ekr.20210312150753.40: *6* << read fake files for unit testing >>
if g.unitTesting:
    p2 = p.copy()
    while(p2.hasNext()):
        p2.moveToNext()
        self.tangle_output[p2.h] = p2.b
#@+node:ekr.20210312150753.41: *5* untangleAll
def untangleAll(self, event=None):
    c = self.c
    self.initUntangleCommand()
    has_roots = False
    for p in c.rootPosition().self_and_siblings():
        ok = self.untangleTree(p, False)
        if ok: has_roots = True
    c.redraw()
    self.errors += g.app.scanErrors
    if not has_roots:
        self.warning("----- the outline contains no roots")
    elif self.errors > 0:
        self.warning("----- untangle command halted because of errors")
    else:
        if not g.unitTesting:
            g.es("untangle complete")
#@+node:ekr.20210312150753.42: *5* untangleMarked
def untangleMarked(self, event=None):
    c = self.c; p = c.rootPosition()
    self.initUntangleCommand()
    marked_flag = False
    while p: # Don't use an iterator.
        if p.isMarked():
            ok = self.untangleTree(p, report_errors=False)
            if ok: marked_flag = True
            if self.errors + g.app.scanErrors > 0: break
            p.moveToNodeAfterTree()
        else:
            p.moveToThreadNext()
    c.redraw()
    self.errors += g.app.scanErrors
    if not marked_flag:
        self.warning("----- the outline contains no marked roots")
    elif self.errors > 0:
        self.warning("----- untangle command halted because of errors")
    else:
        if not g.unitTesting:
            g.es("untangle complete")
#@+node:ekr.20210312150753.43: *5* untangleRoot (calls cleanup)
@
This method untangles the derived files in a VNode known to contain at
least one @root directive. The work is done in three passes. The first
pass creates a TST from the Leo tree so that the next pass will know
what comment conventions to use; that pass is performed in
untangleTree. The second pass creates the UST by scanning the derived
file. The third pass updates the outline using the UST and a new TST
that is created during the pass.

We assume that all sections from root to end are contained in the
derived file, and we attempt to update all such sections. The
begin/end params indicate the range of nodes to be scanned when
building the TST.
@c

def untangleRoot(self, root, begin, end):

    c = self.c
    << return if @silent >>
    s = root.b
    i = 0
    while i < len(s):
        << Set path & root_name to the file specified in the @root directive >>
        path = c.os_path_finalize_join(self.tangle_directory, path)
        if g.unitTesting:
            << fake the file access >>
        else:
            file_buf, e = g.readFileIntoString(path)
        if file_buf is not None:
            file_buf = file_buf.replace('\r', '')
            if not g.unitTesting:
                g.es('', '@root ' + path)
            if 0: # debug
                g.es(self.st_dump())
                g.es("path: " + path)
            section = self.tst[self.root_name]
            assert section
            # Pass 2: Scan the derived files, creating the UST
            self.scan_derived_file(file_buf)
            if self.errors + g.app.scanErrors == 0:
                # Untangle the outline using the UST and a newly-created TST
                << Pass 3 >>
    self.cleanup()
#@+node:ekr.20210312150753.44: *6* << return if @silent >>
if self.print_mode in ("quiet", "silent"):
    g.warning(f"@{self.print_mode} inhibits untangle for {root.h}")
    return
#@+node:ekr.20210312150753.45: *6* << Set path & root_name to the file specified in the @root directive >>
self.root_name = None
while i < len(s):
    code, junk = self.token_type(s, i, report_errors=True)
    i = g.skip_line(s, i)
    if code == at_root:
        # token_type sets root_name unless there is a syntax error.
        if self.root_name: path = self.root_name
        break
if not self.root_name:
    # A bad @root command.  token_type has already given an error.
    self.cleanup()
    return
#@+node:ekr.20210312150753.46: *6* << fake the file access >>
# complications to handle testing of multiple @root directives together with
# @path directives
file_name_path = c.os_path_finalize_join(self.tangle_directory, path)
if (file_name_path.find(c.openDirectory) == 0):
    relative_path = file_name_path[len(c.openDirectory):]
    # don't confuse /u and /usr as having common prefixes
    if (relative_path[: len(os.sep)] == os.sep):
        file_name_path = relative_path[len(os.sep):]
# find the node with the right title, and load self.tangle_output and file_buf
file_buf = self.tangle_output.get(file_name_path)
#@+node:ekr.20210312150753.47: *6* << Pass 3  >>
# This code untangles the root and all its siblings.
# We don't call tangleTree here because we must handle all siblings.
# tanglePass1 handles an entire tree. It also handles @ignore.
self.tangling = False
p = begin
while p and p != end: # Don't use iterator.
    self.scanAllDirectives(p) # sets self.init_delims
    self.tst = {}
    self.untangle_stage1 = False
    self.tanglePass1(p, self.init_delims)
    if self.errors + g.app.scanErrors != 0:
        break
    p.moveToNodeAfterTree()
self.ust_warn_about_orphans()
#@+node:ekr.20210312150753.48: *5* untangleTree
# This funtion is called when the user selects any "Untangle" command.

def untangleTree(self, p, report_errors):
    p = p.copy() # 9/14/04
    c = self.c
    any_root_flag = False
    afterEntireTree = p.nodeAfterTree()
    # Initialize these globals here: they can't be cleared later.
    self.head_root = None
    self.errors = 0; g.app.scanErrors = 0
    c.clearAllVisited() # Used by untangle code.
    self.tangling = False
    self.delims_table = False
    while p and p != afterEntireTree and self.errors + g.app.scanErrors == 0:
        self.setRootFromHeadline(p)
        assert(self.head_root is None)
        theDict = g.get_directives_dict(p, [self.head_root])
        ignore = 'ignore' in theDict
        root = 'root' in theDict
        unit = 'unit' in theDict
        if ignore:
            p.moveToNodeAfterTree()
        elif unit:
            # Expand the context to the @unit directive.
            unitNode = p # 9/27/99
            afterUnit = p.nodeAfterTree()
            self.scanAllDirectives(p) # sets self.init_delims
            p.moveToThreadNext()
            while p and p != afterUnit and self.errors + g.app.scanErrors == 0:
                self.setRootFromHeadline(p)
                assert(self.head_root is None)
                theDict = g.get_directives_dict(p, [self.head_root])
                root = 'root' in theDict
                if root:
                    any_root_flag = True
                    end = None
                    << set end to the next root in the unit >>
                    self.scanAllDirectives(p)
                    self.tanglePass1(p, self.init_delims)
                    self.untangleRoot(p, unitNode, afterUnit)
                    p = end.copy()
                else: p.moveToThreadNext()
        elif root:
            # Limit the range of the @root to its own tree.
            afterRoot = p.nodeAfterTree()
            any_root_flag = True
            self.scanAllDirectives(p)
            # get the delims table
            self.untangle_stage1 = True
            self.tanglePass1(p, self.init_delims)
            self.untangleRoot(p, p, afterRoot)
            p = afterRoot.copy()
        else:
            p.moveToThreadNext()
    self.errors += g.app.scanErrors
    if report_errors:
        if not any_root_flag:
            self.warning("----- The outline contains no roots")
        elif self.errors > 0:
            self.warning("----- Untangle command halted because of errors")
    return any_root_flag
#@+node:ekr.20210312150753.49: *6* << set end to the next root in the unit >>
@
The untangle_root function will untangle an entire tree by calling
untangleTree, so the following code ensures that the next @root node
will not be an offspring of p.
@c
end = p.threadNext()
while end and end != afterUnit:
    flag, i = g.is_special(end.b, "@root")
    if flag and not p.isAncestorOf(end):
        break
    end.moveToThreadNext()
#@+node:ekr.20210312150753.50: *4* tangle
#@+node:ekr.20210312150753.51: *5* Pass 1
#@+node:ekr.20210312150753.52: *6* handle_newline
@
This method handles newline processing while skipping a code section.
It sets 'done' if the line contains an @directive or section
definition that terminates the present code section. On entry: i
should point to the first character of a line. This routine scans past
a line only if it could not contain a section reference.

Returns (i, done)
@c

def handle_newline(self, s, i, delims):
    assert(delims)
    j = i; done = False
    kind, end = self.token_type(s, i, report_errors=False)
    # token_type will not skip whitespace in noweb mode.
    i = g.skip_ws(s, i)
    if kind == plain_line:
        pass
    elif(
        kind == at_code or
        kind == at_doc or
        kind == at_root or
        kind == section_def
    ):
        i = j; done = True # Terminate this code section and rescan.
    elif kind == section_ref:
        # Enter the reference.
        ref = s[i: end]
        self.st_enter_section_name(ref, None, None, None, None)
    elif kind == at_other:
        k = g.skip_to_end_of_line(s, i)
        if g.match_word(s, j, "@language"):
            lang, d1, d2, d3 = g.set_language(s, j)
            delims = (d1, d2, d3)
        elif g.match_word(s, j, "@comment"):
            delims = g.set_delims_from_string(s[j: k])
        i = k
    elif kind == at_chapter or kind == at_section:
        # We don't process chapter or section here
        i = g.skip_to_end_of_line(s, i)
    elif kind == bad_section_name:
        pass
    elif kind == at_web or kind == at_at:
        i += 2 # Skip a CWEB control code.
    else: assert(False)
    return i, done, delims
#@+node:ekr.20210312150753.53: *6* tangleCommands.skip_body & trimTrailingLines
# This method handles all the body text.

def skip_body(self, p, delims):
    << skip_body docstring >>
    # c = self.c
    s = p.b
    code = doc = None; i = 0
    anyChanged = False
    if self.start_mode == "code":
        j = g.skip_blank_lines(s, i)
        i, code, new_delims, reflist = self.skip_code(s, j, delims)
        if code:
            << Define a section for a leading code part >>
        delims = new_delims
    if not code:
        i, doc, delims = self.skip_doc(s, i, delims) # Start in doc section by default.
        if i >= len(s) and doc:
            << Define a section containing only an @doc part >>
    while i < len(s):
        progress = i # progress indicator
        kind, end = self.token_type(s, i, report_errors=True)
        # if g.is_nl(s,i): i = g.skip_nl(s,i)
        i = g.skip_ws(s, i)
        if kind == section_def:
            << Scan and define a section definition >>
        elif kind == at_code:
            i = g.skip_line(s, i)
            << Scan and define an @code defininition >>
        elif kind == at_root:
            i = g.skip_line(s, i)
            << Scan and define a root section >>
        elif kind in (at_doc, at_chapter, at_section):
            i = g.skip_line(s, i)
            i, more_doc, delims = self.skip_doc(s, i, delims)
            doc = doc + more_doc
        else:
            i = g.skip_line(s, i)
        assert(progress < i) # we must make progress!
    # Only call trimTrailingLines if we have changed its body.
    if anyChanged:
        self.trimTrailingLines(p)
    return delims
#@+node:ekr.20210312150753.54: *7* << skip_body docstring >>
'''
The following subsections contain the interface between the Tangle and
Untangle commands. This interface is an important hack, and allows
Untangle to avoid duplicating the logic in skip_tree and its allies.

The aha is this: just at the time the Tangle command enters a
definition into the symbol table, all the information is present that
Untangle needs to update that definition.

To get whitespace exactly right we retain the outline's leading
whitespace and remove leading whitespace from the updated definition.
'''
#@+node:ekr.20210312150753.55: *7* << Define a section for a leading code part >>
if self.header_name:
    # Tangle code.
    part = self.st_enter_section_name(
        self.header_name, code, doc, delims, new_delims)
    if not self.tangling:
        # Untangle code.
        if self.untangle_stage1:
            section = self.st_lookup(self.header_name)
            section.parts[part - 1].reflist(refs=reflist)
        else:
            head = s[: j]; tail = s[i:]
            s, i, changed = self.update_def(self.header, part, head, code, tail)
            if changed: anyChanged = True
    code = doc = None
# leading code without a header name gets silently dropped
#@+node:ekr.20210312150753.56: *7* << Define a section containing only an @doc part >>
@
It's valid for an @doc directive to appear under a headline that does
not contain a section name. In that case, no section is defined.
@c
if self.header_name:
    # Tangle code.
    part = self.st_enter_section_name(self.header_name, code, doc, delims, delims)
    # Untangle code.
    if not self.tangling:
        # Untangle no longer updates doc parts.
        # 12/03/02: Mark the part as having been updated to suppress warning.
        junk, junk = self.ust_lookup(self.header_name, part, update_flag=True)
doc = None
#@+node:ekr.20210312150753.57: *7* << Scan and define a section definition >>
# We enter the code part and any preceding doc part into the symbol table.
# Skip the section definition line.
k = i; i, kind, junk = self.skip_section_name(s, i)
section_name = s[k: i]
assert(kind == section_def)
i = g.skip_to_end_of_line(s, i)
# Tangle code: enter the section name even if the code part is empty.
<<process normal section>>
code = None
doc = ''
#@+node:ekr.20210312150753.58: *8* <<process normal section>>
# Tangle code.
j = g.skip_blank_lines(s, i)
i, code, new_delims, reflist = self.skip_code(s, j, delims)
part = self.st_enter_section_name(section_name, code, doc, delims, new_delims)
delims = new_delims
# Untangle code
if not self.tangling:
    # part may be zero if there was an empty code section (doc part only)
    # In untangle stage1 such code produces no reference list,
    #    thus nothing to do.
    # In untangle stage2, such code cannot be updated because it
    # was either not emitted to the external file or emitted as a doc part only
    #     in either case, there is no code section to update, and we don't
    #     update doc parts.
    if part > 0:
        if self.untangle_stage1:
            section = self.st_lookup(section_name)
            section.parts[part - 1].reflist(refs=reflist)
        else:
            head = s[: j]; tail = s[i:]
            s, i, changed = self.update_def(section_name, part, head, code, tail)
            if changed: anyChanged = True
#@+node:ekr.20210312150753.59: *7* << Scan and define an @code defininition >>
# All @c or @code directives denote < < headline_name > > =
if self.header_name:
    section_name = self.header_name
    <<process normal section>>
else:
    self.error("@c expects the headline: " + self.header + " to contain a section name")
code = None
doc = ''
#@+node:ekr.20210312150753.60: *8* <<process normal section>>
# Tangle code.
j = g.skip_blank_lines(s, i)
i, code, new_delims, reflist = self.skip_code(s, j, delims)
part = self.st_enter_section_name(section_name, code, doc, delims, new_delims)
delims = new_delims
# Untangle code
if not self.tangling:
    # part may be zero if there was an empty code section (doc part only)
    # In untangle stage1 such code produces no reference list,
    #    thus nothing to do.
    # In untangle stage2, such code cannot be updated because it
    # was either not emitted to the external file or emitted as a doc part only
    #     in either case, there is no code section to update, and we don't
    #     update doc parts.
    if part > 0:
        if self.untangle_stage1:
            section = self.st_lookup(section_name)
            section.parts[part - 1].reflist(refs=reflist)
        else:
            head = s[: j]; tail = s[i:]
            s, i, changed = self.update_def(section_name, part, head, code, tail)
            if changed: anyChanged = True
#@+node:ekr.20210312150753.61: *7* << Scan and define a root section >>
# We save the file name in case another @root ends the code section.
old_root_name = self.root_name
#
# Tangle code.
j = g.skip_blank_lines(s, i)
k, code, new_delims, reflist = self.skip_code(s, j, delims)
self.st_enter_root_name(old_root_name, code, doc, delims, new_delims)
delims = new_delims
if not self.tangling:
    # Untangle code.
    if self.untangle_stage1:
        root_section = self.st_lookup(old_root_name)
        assert(root_section)
        root_first_part = root_section.parts[0]
        assert(root_first_part)
        root_first_part.reflist(refs=reflist)
    else:
        part = 1 # Use 1 for root part.
        head = s[: j]; tail = s[k:]
        s, i, changed = self.update_def(old_root_name, part, head, code, tail, is_root_flag=True)
        if changed: anyChanged = True
code = None
doc = ''
#@+node:ekr.20210312150753.62: *7* tangleCommands.trimTrailingLines
def trimTrailingLines(self, p):
    """Trims trailing blank lines from a node.

    It is surprising difficult to do this during Untangle."""
    ### c = self
    body = p.b
    lines = body.split('\n')
    i = len(lines) - 1; changed = False
    while i >= 0:
        line = lines[i]
        j = g.skip_ws(line, 0)
        if j + 1 == len(line):
            del lines[i]
            i -= 1; changed = True
        else: break
    if changed:
        p.b = ''.join(body) + '\n'  # Add back one last newline.
        # Don't set the dirty bit: it would just be annoying.
#@+node:ekr.20210312150753.63: *6* skip_code
@
This method skips an entire code section. The caller is responsible
for entering the completed section into the symbol table. On entry, i
points at the line following the @directive or section definition that
starts a code section. We skip code until we see the end of the body
text or the next @ directive or section defintion that starts a code
or doc part.
@c

def skip_code(self, s, i, delims):
    reflist = []
    code1 = i
    nl_i = i # For error messages
    done = False # True when end of code part seen.
    << skip a noweb code section >>
    code = s[code1: i]
    return i, code, delims, reflist
#@+node:ekr.20210312150753.64: *7* << skip a noweb code section >>
@
This code handles the following escape conventions: double at-sign at
the start of a line and at-<< and at.>.
@c
i, done, delims = self.handle_newline(s, i, delims)
while not done and i < len(s):
    ch = s[i]
    if g.is_nl(s, i):
        nl_i = i = g.skip_nl(s, i)
        i, done, delims = self.handle_newline(s, i, delims)
    elif ch == '@' and (
        g.match(s, i + 1, "<<") or # must be on different lines
        g.match(s, i + 1, ">>")
    ):
        i += 3 # skip the noweb escape sequence.
    elif ch == '<':
        << handle possible noweb section reference >>
    else: i += 1
#@+node:ekr.20210312150753.65: *8* << handle possible noweb section reference >>
j, kind, end = self.is_section_name(s, i)
if kind == section_def:
    k = g.skip_to_end_of_line(s, i)
    # We are in the middle of a line.
    i += 1
    self.error("chunk definition not valid here\n" + s[nl_i: k])
elif kind == bad_section_name:
    i += 1 # This is not an error.  Just skip the '<'.
else:
    assert(kind == section_ref)
    # Enter the reference into the symbol table.
    # Appropriate comment delimiters get specified
    # at the time the section gets defined.
    name = s[i: end]
    self.st_enter_section_name(name, None, None, None, None)
    reflist.append(self.st_lookup(name))
    i = end
#@+node:ekr.20210312150753.66: *6* skip_doc
def skip_doc(self, s, i, delims):
    # Skip @space, @*, @doc, @chapter and @section directives.
    doc1 = i
    while i < len(s):
        if g.is_nl(s, i):
            doc1 = i = g.skip_nl(s, i)
        elif g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@*"):
            i = g.skip_ws(s, i + 2); doc1 = i
        elif g.match(s, i, "@\n"):
            i += 1; doc1 = i
        elif(g.match_word(s, i, "@doc") or
              g.match_word(s, i, "@chapter") or
              g.match_word(s, i, "@section")):
            doc1 = i = g.skip_line(s, i)
        else: break
    while i < len(s):
        kind, junk = self.token_type(s, i, report_errors=False)
        if kind == at_code or kind == at_root or kind == section_def:
            break
        # @language and @comment are honored within document parts
        k = g.skip_line(s, i)
        if kind == at_other:
            if g.match_word(s, i, "@language"):
                lang, d1, d2, d3 = g.set_language(s, i)
                delims = (d1, d2, d3)
            elif g.match_word(s, i, "@comment"):
                delims = g.set_delims_from_string(s[i: k])
        i = k
    doc = s[doc1: i]
    return i, doc, delims
#@+node:ekr.20210312150753.67: *6* skip_headline
@
This function sets ivars that keep track of the indentation level. We
also remember where the next line starts because it is assumed to be
the first line of a documentation section.

A headline can contain a leading section name. If it does, we
substitute the section name if we see an @c directive in the body
text.
@c

def skip_headline(self, p):
    self.header = s = p.h
    # Set self.header_name.
    j = i = g.skip_ws(s, 0)
    i, kind, end = self.is_section_name(s, i)
    if kind == bad_section_name:
        self.header_name = None
    else:
        self.header_name = s[j: end]
#@+node:ekr.20210312150753.68: *5* Pass 2
#@+node:ekr.20210312150753.69: *6* oblank, oblanks, os, otab, otabs (Tangle)
def oblank(self):
    self.oblanks(1)

def oblanks(self, n):
    if abs(n) > 0:
        s = g.toEncodedString(' ' * abs(n), encoding=self.encoding)
        self.output_file.write(s)

def onl(self):
    s = self.output_newline
    s = g.toEncodedString(s, self.encoding, reportErrors=True)
    self.output_file.write(s)

def os(self, s):
    s = s.replace('\r', '\n')
    s = g.toEncodedString(s, self.encoding, reportErrors=True)
    self.output_file.write(s)

def otab(self):
    self.otabs(1)

def otabs(self, n):
    if abs(n) > 0:
        s = g.toEncodedString('\t' * abs(n), self.encoding, reportErrors=True)
        self.output_file.write(s)
#@+node:ekr.20210312150753.70: *6* tangle.put_all_roots
@
This is the top level method of the second pass. It creates a separate derived file
for each @root directive in the outline. The file is actually written only if
the new version of the file is different from the old version,or if the file did
not exist previously. If changed_only_flag FLAG is True only changed roots are
actually written.
@c

def put_all_roots(self):
    c = self.c; outline_name = c.mFileName
    for section in self.root_list:
        file_name = c.os_path_finalize_join(self.tangle_directory, section.name)
        mode = c.config.output_newline
        textMode = mode == 'platform'
        if g.unitTesting:
            self.output_file = g.FileLikeObject()
            temp_name = 'temp-file'
        else:
            self.output_file, temp_name = g.create_temp_file(textMode=textMode)
        if not temp_name:
            g.es("can not create temp file")
            break
        <<Get root specific attributes>>
        <<Put @first lines>>
        if self.use_header_flag and self.print_mode == "verbose":
            << Write a banner at the start of the output file >>
        for part in section.parts:
            if part.is_root:
                self.tangle_indent = 0 # Initialize global.
                self.put_PartNode(part, False) # output first lws
        self.onl() # Make sure the file ends with a cr/lf
        << unit testing fake files>>
        self.output_file.close()
        self.output_file = None
        << unit testing set result and continue >>
        if self.errors + g.app.scanErrors == 0:
            self.update_file_if_changed(c, file_name, temp_name)
        else:
            g.es("unchanged:", file_name)
            << Erase the temporary file >>
#@+node:ekr.20210312150753.71: *7* <<Get root specific attributes>> (Tangle)
# Stephen Schaefer, 9/2/02
# Retrieve the full complement of state for the root node
self.language = section.RootAttributes.language
self.use_header_flag = section.RootAttributes.use_header_flag
self.print_mode = section.RootAttributes.print_mode
self.path = section.RootAttributes.path
self.page_width = section.RootAttributes.page_width
self.tab_width = section.RootAttributes.tab_width
# Stephen P. Schaefer, 9/13/2002
self.first_lines = section.RootAttributes.first_lines
#@+node:ekr.20210312150753.72: *7* <<Put @first lines>>
# Stephen P. Schaefer 9/13/2002
if self.first_lines:
    self.os(self.first_lines)
#@+node:ekr.20210312150753.73: *7* <<Write a banner at the start of the output file>>
# a root section must have at least one part
assert section.parts
delims = section.parts[0].delims
if delims[0]:
    self.os(delims[0])
    self.os(" Created by Leo from: ")
    self.os(outline_name)
    self.onl(); self.onl()
elif delims[1] and delims[2]:
    self.os(delims[1])
    self.os(" Created by Leo from: ")
    self.os(outline_name)
    self.oblank(); self.os(delims[2])
    self.onl(); self.onl()
#@+node:ekr.20210312150753.74: *7* << unit testing fake files>>
if g.unitTesting:
    # complications to handle testing of multiple @root directives together with
    # @path directives
    file_name_path = file_name
    if (file_name_path.find(c.openDirectory) == 0):
        relative_path = file_name_path[len(c.openDirectory):]
        # don't confuse /u and /usr as having common prefixes
        if (relative_path[: len(os.sep)] == os.sep):
            file_name_path = relative_path[len(os.sep):]
    self.tangle_output[file_name_path] = self.output_file.get()
#@+node:ekr.20210312150753.75: *7* << unit testing set result and continue >>
if g.unitTesting:
    assert self.errors == 0
    g.app.unitTestDict['tangle'] = True
    g.app.unitTestDict['tangle_directory'] = self.tangle_directory
    if g.app.unitTestDict.get('tangle_output_fn'):
        g.app.unitTestDict['tangle_output_fn'] += "\n" + file_name
    else:
        g.app.unitTestDict['tangle_output_fn'] = file_name
    continue
#@+node:ekr.20210312150753.76: *7* << Erase the temporary file >>
try: # Just delete the temp file.
    os.remove(temp_name)
except Exception:
    pass
#@+node:ekr.20210312150753.77: *6* put_code
@
This method outputs a code section, expanding section references by
their definition. We should see no @directives or section definitions
that would end the code section.

Most of the differences bewteen noweb mode and CWEB mode are handled
by token_type(called from put_newline). Here, the only difference is
that noweb handles double-@ signs only at the start of a line.
@c

def put_code(self, s, no_first_lws_flag, delims):

    i = 0
    if i < len(s):
        i = self.put_newline(s, i, no_first_lws_flag)
        # Double @ is valid in both noweb and CWEB modes here.
        if g.match(s, i, "@@"):
            self.os('@'); i += 2
    while i < len(s):
        progress = i
        ch = s[i]
        if g.match(s, i, "<<"):
            << put possible section reference >>
        elif ch == '@': # We are in the middle of a line.
            << handle noweb @ < < convention >>
        elif ch == '\r':
            i += 1
        elif ch == '\n':
            i += 1; self.onl()
            << elide @comment or @language >>
            i = self.put_newline(s, i, False) # Put full lws
        else: self.os(s[i]); i += 1
        assert(progress < i)
#@+node:ekr.20210312150753.78: *7* <<put possible section reference >>
j, kind, name_end = self.is_section_name(s, i)
if kind == section_def:
    # We are in the middle of a code section
    self.error(
        "Should never happen:\n" +
        "section definition while putting a section reference: " +
        s[i: j])
    i += 1
elif kind == bad_section_name:
    self.os(s[i]); i += 1 # This is not an error.
else:
    assert(kind == section_ref)
    name = s[i: name_end]
    self.put_section(s, i, name, name_end, delims)
    i = j
#@+node:ekr.20210312150753.79: *7* << handle noweb @ < < convention >>
@
The user must ensure that neither @ < < nor @ > > occurs in comments
or strings. However, it is valid for @ < < or @ > > to appear in the
doc chunk or in a single-line comment.
@c
if g.match(s, i, "@<<"):
    self.os("/*@*/<<"); i += 3
elif g.match(s, i, "@>>"):
    self.os("/*@*/>>"); i += 3
else: self.os("@"); i += 1
#@+node:ekr.20210312150753.80: *7* << elide @comment or @language >>
while g.match_word(s, i, "@comment") or g.match_word(s, i, "@language"):
    i = g.skip_line(s, i)
#@+node:ekr.20210312150753.81: *6* put_doc
# This method outputs a doc section within a block comment.

def put_doc(self, s, delims):

    width = self.page_width
    words = 0; word_width = 0; line_width = 0
    # 8/1/02: can't use choose here!
    if delims[0] is None: single_w = 0
    else: single_w = len(delims[0])
    # Make sure we put at least 20 characters on a line.
    if width - max(0, self.tangle_indent) < 20:
        width = max(0, self.tangle_indent) + 20
    # Skip Initial white space in the doc part.
    i = g.skip_ws_and_nl(s, 0)
    if i < len(s) and (self.print_mode == "verbose" or self.print_mode == "quiet"):
        use_block_comment = delims[1] and delims[2]
        use_single_comment = not use_block_comment and delims[0]
        # javadoc_comment = use_block_comment and delims[1] == "/**"
        if use_block_comment or use_single_comment:
            self.put_leading_ws(self.tangle_indent)
            if use_block_comment:
                self.os(delims[1])
            << put the doc part >>
            self.onl()
            self.put_leading_ws(self.tangle_indent)
            if use_block_comment:
                self.os(delims[2])
            self.onl()
        else: self.onl()
#@+node:ekr.20210312150753.82: *7* <<put the doc part>>
@
This code fills and outputs each line of a doc part. It keeps track of
whether the next word will fit on a line,and starts a new line if
needed.
@c
if use_single_comment:
    self.os(delims[0]); self.otab()
    line_width = (single_w / abs(self.tab_width) + 1) * abs(self.tab_width)
else:
    line_width = abs(self.tab_width)
    self.onl(); self.otab()
self.put_leading_ws(self.tangle_indent)
line_width += max(0, self.tangle_indent)
words = 0; word_width = 0
while i < len(s):
    <<output or skip whitespace or newlines>>
    if i >= len(s):
        break
    j = i; word_width = 0
    while j < len(s) and not g.is_ws_or_nl(s, j):
        word_width += 1
        j += 1
    if words == 0 or line_width + word_width < width:
        words += 1
        self.os(s[i: j])
        i = j
        line_width += word_width
    else:
        # 11-SEP-2002 DTHEIN: Fixed linewrapping bug in
        # tab-then-comment sequencing
        self.onl()
        if use_single_comment:
            self.os(delims[0]); self.otab()
            line_width = (single_w / abs(self.tab_width) + 1) * abs(self.tab_width)
        else:
            self.otab()
            line_width = abs(self.tab_width)
        words = 0
        self.put_leading_ws(self.tangle_indent)
        # tangle_indent is in spaces.
        line_width += max(0, self.tangle_indent)
#@+node:ekr.20210312150753.83: *8* <<output or skip whitespace or newlines>>
@
This outputs whitespace if it fits, and ignores it otherwise, and
starts a new line if a newline is seen. The effect of self code is
that we never start a line with whitespace that was originally at the
end of a line.
@c
while g.is_ws_or_nl(s, i):
    ch = s[i]
    if ch == '\t':
        pad = abs(self.tab_width) - (line_width % abs(self.tab_width))
        line_width += pad
        if line_width < width: self.otab()
        i += 1
    elif ch == ' ':
        line_width += 1
        if line_width < width: self.os(ch)
        i += 1
    else:
        assert(g.is_nl(s, i))
        self.onl()
        if use_single_comment:
            # New code: 5/31/00
            self.os(delims[0]); self.otab()
            line_width = (single_w / abs(self.tab_width) + 1) * abs(self.tab_width)
        else:
            self.otab()
            line_width = abs(self.tab_width)
        i = g.skip_nl(s, i)
        words = 0
        self.put_leading_ws(self.tangle_indent)
        # tangle_indent is in spaces.
        line_width += max(0, self.tangle_indent)
#@+node:ekr.20210312150753.84: *6* put_leading_ws
# Puts tabs and spaces corresponding to n spaces, assuming that we are at the start of a line.

def put_leading_ws(self, n):

    w = self.tab_width
    if w > 1:
        q, r = divmod(n, w)
        self.otabs(q)
        self.oblanks(r)
    else:
        self.oblanks(n)
#@+node:ekr.20210312150753.85: *6* put_newline
@
This method handles scanning when putting the start of a new line.
Unlike the corresponding method in pass one, this method doesn't need
to set a done flag in the caller because the caller already knows
where the code section ends.
@c

def put_newline(self, s, i, no_first_lws_flag):
    kind, junk = self.token_type(s, i, report_errors=False)
    << Output leading white space except for blank lines >>
    if i >= len(s):
        return i
    if kind == at_web or kind == at_at:
        i += 2 # Allow the line to be scanned.
    elif kind == at_doc or kind == at_code:
        pass
    else:
        # These should have set limit in pass 1.
        assert(kind != section_def and kind != at_chapter and kind != at_section)
    return i
#@+node:ekr.20210312150753.86: *7* << Output leading white space except for blank lines >>
j = i; i = g.skip_ws(s, i)
if i < len(s) and not g.is_nl(s, i):
    # Conditionally output the leading previous leading whitespace.
    if not no_first_lws_flag:
        self.put_leading_ws(self.tangle_indent)
    # Always output the leading whitespace of _this_ line.
    k, width = g.skip_leading_ws_with_indent(s, j, self.tab_width)
    self.put_leading_ws(width)
#@+node:ekr.20210312150753.87: *6* put_PartNode
# This method outputs one part of a section definition.

def put_PartNode(self, part, no_first_lws_flag):

    if part.doc and self.output_doc_flag and self.print_mode != "silent":
        self.put_doc(part.doc, part.delims)
    if part.code:
        # comment convention cannot change in the middle of a doc part
        self.put_code(part.code, no_first_lws_flag, part.delims)
#@+node:ekr.20210312150753.88: *6* put_section
@
This method outputs the definition of a section and all sections
referenced from the section. name is the section's name. This code
checks for recursive definitions by calling section_check(). We can
not allow section x to expand to code containing another call to
section x, either directly or indirectly.
@c

def put_section(self, s, i, name, name_end, delims):
    
    j = g.skip_line(s, i)
    outer_old_indent = self.tangle_indent
    trailing_ws_indent = 0 # Set below.
    inner_old_indent = 0 # Set below.
    newline_flag = False # True if the line ends with the reference.
    assert(g.match(name, 0, "<<") or g.match(name, 0, "@<"))
    << Calculate the new value of tangle_indent >>
    << Set 'newline_flag' if the line ends with the reference >>
    section = self.st_lookup(name)
    if section and section.parts:
        # Expand the section only if we are not already expanding it.
        if self.section_check(name):
            self.section_stack.append(name)
            << put all parts of the section definition >>
            self.section_stack.pop()
    else:
        << Put a comment about the undefined section >>
    if not newline_flag:
        self.put_leading_ws(trailing_ws_indent)
    self.tangle_indent = outer_old_indent
    return i, name_end
#@+node:ekr.20210312150753.89: *7* << Calculate the new value of tangle_indent >>
# Find the start of the line containing the reference.
j = i
while j > 0 and not g.is_nl(s, j):
    j -= 1
if g.is_nl(s, j):
    j = g.skip_nl(s, j)
# Bump the indentation
j, width = g.skip_leading_ws_with_indent(s, j, self.tab_width)
self.tangle_indent += width
#
# Force no trailing whitespace in @silent mode.
if self.print_mode == "silent":
    trailing_ws_indent = 0
else:
    trailing_ws_indent = self.tangle_indent
# Increase the indentation if the section reference does not immediately follow
# the leading white space.  4/3/01: Make no adjustment in @silent mode.
if (j < len(s) and self.print_mode != "silent" and s[j] != '<'):
    self.tangle_indent += abs(self.tab_width)
#@+node:ekr.20210312150753.90: *7* << Set 'newline_flag' if the line ends with the reference >>
if self.print_mode != "silent":
    i = name_end
    i = g.skip_ws(s, i)
    newline_flag = (i >= len(s) or g.is_nl(s, i))
#@+node:ekr.20210312150753.91: *7* <<put all parts of the section definition>>
@
This section outputs each part of a section definition. We first count
how many parts there are so that the code can output a comment saying
'part x of y'.
@c
# Output each part of the section.
sections = len(section.parts)
count = 0
for part in section.parts:
    count += 1
    # In @silent mode, there is no sentinel line to "use up" the previously output
    # leading whitespace.  We set the flag to tell put_PartNode and put_code
    # not to call put_newline at the start of the first code part of the definition.
    no_first_leading_ws_flag = (count == 1 and self.print_mode == "silent")
    inner_old_indent = self.tangle_indent
    # 4/3/01: @silent inhibits newlines after section expansion.
    if self.print_mode != "silent":
        << Put the section name in a comment >>
    self.put_PartNode(part, no_first_leading_ws_flag)
    # 4/3/01: @silent inhibits newlines after section expansion.
    if count == sections and (self.print_mode != "silent" and self.print_mode != "quiet"):
        << Put the ending comment >>
    # Restore the old indent.
    self.tangle_indent = inner_old_indent
#@+node:ekr.20210312150753.92: *8* << Put the section name in a comment >>
if count > 1:
    self.onl()
    self.put_leading_ws(self.tangle_indent)
# Don't print trailing whitespace
name = name.rstrip()
if part.delims[0]:
    self.os(part.delims[0]); self.oblank(); self.os(name)
    # put (n of m)
    if sections > 1:
        self.oblank()
        self.os(f"{count} of {sections}")
else:
    assert part.delims[1] and part.delims[2]
    self.os(part.delims[1]); self.oblank(); self.os(name)
    # put (n of m)
    if sections > 1:
        self.oblank()
        self.os(f"{count} of {sections}")
    self.oblank(); self.os(part.delims[2])
self.onl() # Always output a newline.
#@+node:ekr.20210312150753.93: *8* << Put the ending comment >>
@
We do not produce an ending comment unless we are ending the last part
of the section,and the comment is clearer if we don't say(n of m).
@c
self.onl(); self.put_leading_ws(self.tangle_indent)
#  Don't print trailing whitespace
while name_end > 0 and g.is_ws(s[name_end - 1]):
    name_end -= 1
if section.delims[0]:
    self.os(section.delims[0]); self.oblank()
    self.os("-- end -- "); self.os(name)
else:
    self.os(section.delims[1]); self.oblank()
    self.os("-- end -- "); self.os(name)
    self.oblank(); self.os(section.delims[2])
@ The following code sets a flag for untangle.

If something follows the section reference we must add a newline,
otherwise the "something" would become part of the comment. Any
whitespace following the (!newline) should follow the section
defintion when Untangled.
@c
if not newline_flag:
    self.os(" (!newline)") # LeoCB puts the leading blank, so we must do so too.
    # Put the whitespace following the reference.
    while name_end < len(s) and g.is_ws(s[name_end]):
        self.os(s[name_end])
        name_end += 1
    self.onl() # We must supply the newline!
#@+node:ekr.20210312150753.94: *7* <<Put a comment about the undefined section>>
self.onl(); self.put_leading_ws(self.tangle_indent)
if self.print_mode != "silent":
    if delims[0]:
        self.os(delims[0])
        self.os(" undefined section: "); self.os(name); self.onl()
    else:
        self.os(delims[1])
        self.os(" undefined section: "); self.os(name)
        self.oblank(); self.os(delims[2]); self.onl()
self.error("Undefined section: " + name)
#@+node:ekr.20210312150753.95: *6* section_check
@
We can not allow a section to be defined in terms of itself, either
directly or indirectly.

We push an entry on the section stack whenever beginning to expand a
section and pop the section stack at the end of each section. This
method checks whether the given name appears in the stack. If so, the
section is defined in terms of itself.
@c

def section_check(self, name):
    if name in self.section_stack:
        s = "Invalid recursive reference of " + name + "\n"
        for n in self.section_stack:
            s += "called from: " + n + "\n"
        self.error(s)
        return False
    return True
#@+node:ekr.20210312150753.96: *4* tst
#@+node:ekr.20210312150753.97: *5* st_check
def st_check(self):
    """Checks the given symbol table for defined but never referenced sections."""
    for name in sorted(self.tst):
        section = self.tst[name]
        if not section.referenced:
            lp = "<< "
            rp = " >>"
            g.es('', ' ' * 4, 'warning:', lp, section.name, rp, 'has been defined but not used.')
#@+node:ekr.20210312150753.98: *5* st_dump
# Dumps the given symbol table in a readable format.

def st_dump(self, verbose_flag=True):
    s = "\ndump of symbol table...\n"
    for name in sorted(self.tst):
        section = self.tst[name]
        if verbose_flag:
            s += self.st_dump_node(section)
        else:
            theType = "  " if section.parts else "un"
            s += ("\n" + theType + "defined:[" + section.name + "]")
        s += "\nsection delims: " + repr(section.delims)
    return s
#@+node:ekr.20210312150753.99: *5* st_dump_node
# Dumps each part of a section's definition.

def st_dump_node(self, section):
    return section.dump()
#@+node:ekr.20210312150753.100: *5* st_enter
# The comment delimiters for the start sentinel are kept in the part;
# for the end sentinel, in the section

def st_enter(self, name, code, doc, delims_begin, delims_end, is_root_flag=False):
    """Enters names and their associated code and doc parts into the given symbol table."""
    section = self.st_lookup(name, is_root_flag)
    assert(section)
    if doc:
        doc = doc.rstrip() # remove trailing lines.
    if code:
        if self.print_mode != "silent": # @silent supresses newline processing.
            i = g.skip_blank_lines(code, 0) # remove leading lines.
            if i > 0: code = code[i:]
            if code: code = code.rstrip() # remove trailing lines.
        if not code: code = None
    if self.tangling and code:
        << check for duplicate code definitions >>
    if code or doc:
        part = PartNode(name, code, doc, is_root_flag, False, delims_begin)
        section.parts.append(part)
        section.delims = delims_end
    else: # A reference
        section.referenced = True
    if is_root_flag:
        self.root_list.append(section)
        section.referenced = True # Mark the root as referenced.
        <<remember root node attributes>>
    return len(section.parts) # part number
#@+node:ekr.20210312150753.101: *6* <<check for duplicate code definitions >>
for part in section.parts:
    if code == part.code:
        s = g.angleBrackets(section.name)
        g.es('warning: possible duplicate definition of:', s)
#@+node:ekr.20210312150753.102: *6* <<remember root node attributes>>
# Stephen Schaefer, 9/2/02
# remember the language and comment characteristics
section.RootAttributes = RootAttributes(self)
#@+node:ekr.20210312150753.103: *5* st_enter_root_name
# Enters a root name into the given symbol table.

def st_enter_root_name(self, name, code, doc, delims_begin, delims_end):

    if name: # User errors can result in an empty @root name.
        self.st_enter(name, code, doc, delims_begin, delims_end, is_root_flag=True)
#@+node:ekr.20210312150753.104: *5* st_enter_section_name
def st_enter_section_name(self, name, code, doc, delims_begin, delims_end):
    """Enters a section name into the given symbol table.

    The code and doc pointers are None for references."""
    return self.st_enter(name, code, doc, delims_begin, delims_end)
#@+node:ekr.20210312150753.105: *5* st_lookup
def st_lookup(self, name, is_root_flag=False):
    """Looks up name in the symbol table and creates a TstNode for it if it does not exist."""
    if is_root_flag:
        key = name
    else:
        key = self.standardize_name(name)
    if key in self.tst:
        section = self.tst[key]
        return section
    section = TstNode(key, is_root_flag)
    self.tst[key] = section
    return section
#@+node:ekr.20210312150753.106: *4* ust
#@+node:ekr.20210312150753.107: *5* ust_dump
def ust_dump(self):
    s = "\n---------- Untangle Symbol Table ----------"
    for name in sorted(self.ust):
        section = self.ust[name]
        s += "\n\n" + section.dump()
    s += "\n--------------------"
    return s
#@+node:ekr.20210312150753.108: *5* ust_enter
def ust_enter(self, name, part, of, code, nl_flag, is_root_flag=False):
    << ust_enter docstring >>
    if not is_root_flag:
        name = self.standardize_name(name)
    << remove blank lines from the start and end of the text >>
    u = UstNode(name, code, part, of, nl_flag, False) # update_flag
    if name not in self.ust:
        self.ust[name] = u
    section = self.ust[name]
    section.parts[part] = u # Parts may be defined in any order.
#@+node:ekr.20210312150753.109: *6* << ust_enter docstring >>
'''
This routine enters names and their code parts into the given table.
The 'part' and 'of' parameters are taken from the "(part n of m)"
portion of the line that introduces the section definition in the C
code.

If no part numbers are given the caller should set the 'part' and 'of'
parameters to zero. The caller is reponsible for checking for
duplicate parts.

This function handles names scanned from a source file; the
corresponding st_enter routine handles names scanned from outlines.
'''
#@+node:ekr.20210312150753.110: *6* << remove blank lines from the start and end of the text >>
i = g.skip_blank_lines(code, 0)
if i > 0:
    code = code[i:].rstrip()
#@+node:ekr.20210312150753.111: *5* ust_lookup
def ust_lookup(self, name, part_number, is_root_flag=False, update_flag=False):
    '''
    Search the given table for a part matching the name and part number.
    '''
    if not is_root_flag:
        name = self.standardize_name(name)
    if part_number == 0: part_number = 1 # A hack: zero indicates the first part.
    if name in self.ust:
        section = self.ust[name]
        if part_number in section.parts:
            part = section.parts[part_number]
            if update_flag: part.update_flag = True
            return part, True
    return None, False
#@+node:ekr.20210312150753.112: *5* ust_warn_about_orphans
def ust_warn_about_orphans(self):
    """Issues a warning about any sections in the derived file for which
    no corresponding section has been seen in the outline."""
    for section in self.ust.values():
        for part in section.parts.values():
            assert(part.of == section.of)
            if not part.update_flag:
                # lp = "<< "
                # rp = " >>"
                # g.es("warning:",'%s%s(%s)%s' % (lp,part.name,part.part,rp),
                  # "is not in the outline")
                break # One warning per section is enough.
#@+node:ekr.20210312150753.113: *4* untangle
#@+node:ekr.20210312150753.114: *5* compare_comments
@
This function compares the interior of comments and returns True if
they are identical except for whitespace or newlines. It is up to the
caller to eliminate the opening and closing delimiters from the text
to be compared.
@c

def compare_comments(self, s1, s2):
    tot_len = 0
    if self.comment: tot_len += len(self.comment)
    if self.comment_end: tot_len += len(self.comment_end)
    p1, p2 = 0, 0
    while p1 < len(s1) and p2 < len(s2):
        p1 = g.skip_ws_and_nl(s1, p1)
        p2 = g.skip_ws_and_nl(s2, p2)
        if self.comment and self.comment_end:
            << Check both parts for @ comment conventions >>
        if p1 >= len(s1) or p2 >= len(s2):
            break
        if s1[p1] != s2[p2]:
            return False
        p1 += 1; p2 += 1
    p1 = g.skip_ws_and_nl(s1, p1)
    p2 = g.skip_ws_and_nl(s2, p2)
    return p1 == len(s1) and p2 == len(s2)
#@+node:ekr.20210312150753.115: *6* << Check both parts for @ comment conventions >>
@
This code is used in forgiving_compare()and in compare_comments().

In noweb mode we allow / * @ * /  (without the spaces)to be equal to @.
We must be careful not to run afoul of this very convention here!
@c
if p1 < len(s1) and s1[p1] == '@':
    if g.match(s2, p2, self.comment + '@' + self.comment_end):
        p1 += 1
        p2 += tot_len + 1
        continue
elif p2 < len(s2) and s2[p2] == '@':
    if g.match(s1, p1, self.comment + '@' + self.comment_end):
        p2 += 1
        p1 += tot_len + 1
        continue
#@+node:ekr.20210312150753.116: *5* forgiving_compare (tangle)
@
This is the "forgiving compare" function. It compares two texts and
returns True if they are identical except for comments or non-critical
whitespace. Whitespace inside strings or preprocessor directives must
match exactly. @language and @comment in the outline version are
ignored. s1 is the outline version, s2 is the external file version.
@c

def forgiving_compare(self, name, part, s1, s2):

    s1 = g.toUnicode(s1, self.encoding)
    s2 = g.toUnicode(s2, self.encoding)
    << Define forgiving_compare vars >>
    p1 = g.skip_ws_and_nl(s1, 0)
    p2 = g.skip_ws_and_nl(s2, 0)
    result = True
    while result and p1 < len(s1) and p2 < len(s2):
        first1 = p1; first2 = p2
        if self.comment and self.comment_end:
            << Check both parts for @ comment conventions >>
        ch1 = s1[p1]
        if ch1 == '\r' or ch1 == '\n':
            << Compare non-critical newlines >>
        elif ch1 == ' ' or ch1 == '\t':
            << Compare non-critical whitespace >>
        elif ch1 == '\'' or ch1 == '"':
            << Compare possible strings >>
        elif ch1 == '#':
            << Compare possible preprocessor directives >>
        elif ch1 == '<':
            # NB: support for derived noweb or CWEB file
            << Compare possible section references >>
        elif ch1 == '@':
            << Skip @language or @comment in outline >>
        else:
            << Compare comments or single characters >>
    << Make sure both parts have ended >>
    return result
#@+node:ekr.20210312150753.117: *6* << Define forgiving_compare vars >>
# scan_derived_file has set the ivars describing comment delims.
first1 = first2 = 0
tot_len = 0
if self.comment: tot_len += len(self.comment)
if self.comment_end: tot_len += len(self.comment_end)
#@+node:ekr.20210312150753.118: *6* << Check both parts for @ comment conventions >>
@
This code is used in forgiving_compare()and in compare_comments().

In noweb mode we allow / * @ * /  (without the spaces)to be equal to @.
We must be careful not to run afoul of this very convention here!
@c
if p1 < len(s1) and s1[p1] == '@':
    if g.match(s2, p2, self.comment + '@' + self.comment_end):
        p1 += 1
        p2 += tot_len + 1
        continue
elif p2 < len(s2) and s2[p2] == '@':
    if g.match(s1, p1, self.comment + '@' + self.comment_end):
        p2 += 1
        p1 += tot_len + 1
        continue
#@+node:ekr.20210312150753.119: *6* << Compare non-critical newlines >>
p1 = g.skip_ws_and_nl(s1, p1)
p2 = g.skip_ws_and_nl(s2, p2)
#@+node:ekr.20210312150753.120: *6* << Compare non-critical whitespace >>
p1 = g.skip_ws(s1, p1)
p2 = g.skip_ws(s2, p2)
#@+node:ekr.20210312150753.121: *6* << Compare possible strings >>
# This code implicitly assumes that string1_len == string2_len == 1.
# The match test ensures that the language actually supports strings.
if (g.match(s1, p1, self.string1) or g.match(s1, p1, self.string2)) and s1[p1] == s2[p2]:
    if self.language == "pascal":
        << Compare Pascal strings >>
    else:
        << Compare C strings >>
    if not result:
        self.mismatch("Mismatched strings")
else:
    << Compare single characters >>
#@+node:ekr.20210312150753.122: *7* << Compare Pascal strings >>
@
We assume the Pascal string is on a single line so the problems with
cr/lf do not concern us.
@c
first1 = p1; first2 = p2
p1 = g.skip_pascal_string(s1, p1)
p2 = g.skip_pascal_string(s2, p2)
result = s1[first1, p1] == s2[first2, p2]
#@+node:ekr.20210312150753.123: *7* << Compare C strings >>
delim = s1[p1]
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
while result and p1 < len(s1) and p2 < len(s2):
    if s1[p1] == delim and self.is_end_of_string(s1, p1, delim):
        result = (s2[p2] == delim and self.is_end_of_string(s2, p2, delim))
        p1 += 1; p2 += 1
        break
    elif g.is_nl(s1, p1) and g.is_nl(s2, p2):
        p1 = g.skip_nl(s1, p1)
        p2 = g.skip_nl(s2, p2)
    else:
        result = s1[p1] == s2[p2]
        p1 += 1; p2 += 1
#@+node:ekr.20210312150753.124: *7* << Compare single characters >>
assert(p1 < len(s1) and p2 < len(s2))
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
if not result: self.mismatch("Mismatched single characters")
#@+node:ekr.20210312150753.125: *6* << Compare possible preprocessor directives >>
if self.language == "c":
    << compare preprocessor directives >>
else:
    << compare single characters >>
#@+node:ekr.20210312150753.126: *7* << Compare preprocessor directives >>
# We cannot assume that newlines are single characters.
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
while result and p1 < len(s1) and p2 < len(s2):
    if g.is_nl(s1, p1):
        result = g.is_nl(s2, p2)
        if not result or self.is_end_of_directive(s1, p1):
            break
        p1 = g.skip_nl(s1, p1)
        p2 = g.skip_nl(s2, p2)
    else:
        result = s1[p1] == s2[p2]
        p1 += 1; p2 += 1
if not result:
    self.mismatch("Mismatched preprocessor directives")
#@+node:ekr.20210312150753.127: *7* << Compare single characters >>
assert(p1 < len(s1) and p2 < len(s2))
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
if not result: self.mismatch("Mismatched single characters")
#@+node:ekr.20210312150753.128: *6* << Compare possible section references >>
if s1[p1] == '<': start_ref = "<<"
else: start_ref = None
# Tangling may insert newlines.
p2 = g.skip_ws_and_nl(s2, p2)
junk, kind1, junk2 = self.is_section_name(s1, p1)
junk, kind2, junk2 = self.is_section_name(s2, p2)
if start_ref and (kind1 != bad_section_name or kind2 != bad_section_name):
    result = self.compare_section_names(s1[p1:], s2[p2:])
    if result:
        p1, junk1, junk2 = self.skip_section_name(s1, p1)
        p2, junk1, junk2 = self.skip_section_name(s2, p2)
    else: self.mismatch("Mismatched section names")
else:
    # Neither p1 nor p2 points at a section name.
    result = s1[p1] == s2[p2]
    p1 += 1; p2 += 1
    if not result:
        self.mismatch("Mismatch at '@' or '<'")
#@+node:ekr.20210312150753.129: *6* << Skip @language or @comment in outline >>
if g.match_word(s1, p1 + 1, "language") or g.match_word(s1, p1 + 1, "comment"):
    p1 = g.skip_line(s1, p1 + 7)
else:
    << Compare single characters >>
#@+node:ekr.20210312150753.130: *7* << Compare single characters >>
assert(p1 < len(s1) and p2 < len(s2))
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
if not result: self.mismatch("Mismatched single characters")
#@+node:ekr.20210312150753.131: *6* << Compare comments or single characters >>
if g.match(s1, p1, self.sentinel) and g.match(s2, p2, self.sentinel):
    first1 = p1; first2 = p2
    p1 = g.skip_to_end_of_line(s1, p1)
    p2 = g.skip_to_end_of_line(s2, p2)
    result = self.compare_comments(s1[first1: p1], s2[first2: p2])
    if not result:
        self.mismatch("Mismatched sentinel comments")
elif g.match(s1, p1, self.line_comment) and g.match(s2, p2, self.line_comment):
    first1 = p1; first2 = p2
    p1 = g.skip_to_end_of_line(s1, p1)
    p2 = g.skip_to_end_of_line(s2, p2)
    result = self.compare_comments(s1[first1: p1], s2[first2: p2])
    if not result:
        self.mismatch("Mismatched single-line comments")
elif g.match(s1, p1, self.comment) and g.match(s2, p2, self.comment):
    while(
        p1 < len(s1) and p2 < len(s2) and
        not g.match(s1, p1, self.comment_end) and
        not g.match(s2, p2, self.comment_end)
    ):
        # ws doesn't have to match exactly either!
        if g.is_nl(s1, p1) or g.is_ws(s1[p1]):
            p1 = g.skip_ws_and_nl(s1, p1)
        else: p1 += 1
        if g.is_nl(s2, p2) or g.is_ws(s2[p2]):
            p2 = g.skip_ws_and_nl(s2, p2)
        else: p2 += 1
    p1 = g.skip_ws_and_nl(s1, p1)
    p2 = g.skip_ws_and_nl(s2, p2)
    if g.match(s1, p1, self.comment_end) and g.match(s2, p2, self.comment_end):
        first1 = p1; first2 = p2
        p1 += len(self.comment_end)
        p2 += len(self.comment_end)
        result = self.compare_comments(s1[first1: p1], s2[first2: p2])
    else: result = False
    if not result:
        self.mismatch("Mismatched block comments")
elif g.match(s1, p1, self.comment2) and g.match(s2, p2, self.comment2):
    while(
        p1 < len(s1) and p2 < len(s2) and
        not g.match(s1, p1, self.comment2_end) and
        not g.match(s2, p2, self.comment2_end)
    ):
        # ws doesn't have to match exactly either!
        if g.is_nl(s1, p1) or g.is_ws(s1[p1]):
            p1 = g.skip_ws_and_nl(s1, p1)
        else: p1 += 1
        if g.is_nl(s2, p2) or g.is_ws(s2[p2]):
            p2 = g.skip_ws_and_nl(s2, p2)
        else: p2 += 1
    p1 = g.skip_ws_and_nl(s1, p1)
    p2 = g.skip_ws_and_nl(s2, p2)
    if g.match(s1, p1, self.comment2_end) and g.match(s2, p2, self.comment2_end):
        first1 = p1; first2 = p2
        p1 += len(self.comment2_end)
        p2 += len(self.comment2_end)
        result = self.compare_comments(s1[first1: p1], s2[first2: p2])
    else: result = False
    if not result:
        self.mismatch("Mismatched alternate block comments")
else:
    << Compare single characters >>
#@+node:ekr.20210312150753.132: *7* << Compare single characters >>
assert(p1 < len(s1) and p2 < len(s2))
result = s1[p1] == s2[p2]
p1 += 1; p2 += 1
if not result: self.mismatch("Mismatched single characters")
#@+node:ekr.20210312150753.133: *6* << Make sure both parts have ended >>
if result:
    p1 = g.skip_ws_and_nl(s1, p1)
    p2 = g.skip_ws_and_nl(s2, p2)
    result = p1 >= len(s1) and p2 >= len(s2)
    if not result:
        # Show the ends of both parts.
        p1 = len(s1)
        p2 = len(s2)
        self.mismatch("One part ends before the other.")
#@+node:ekr.20210312150753.134: *5* mismatch
def mismatch(self, message):
    self.message = message
#@+node:ekr.20210312150753.135: *5* scan_derived_file (pass 2)
@

This function scans an entire derived file in s, discovering section or part
definitions.

This is the easiest place to delete leading whitespace from each line: we simply
don't copy it. We also ignore leading blank lines and trailing blank lines. The
resulting definition must compare equal using the "forgiving" compare to any
other definitions of that section or part.

We use a stack to handle nested expansions. The outermost level of expansion
corresponds to the @root directive that created the file. When the stack is
popped, the indent variable is restored.

self.root_name is the name of the file mentioned in the @root directive.

The caller has deleted all body_ignored_newlines from the text.
@c

def scan_derived_file(self, s):

    c = self.c
    self.def_stack = []
    << set the private global matching vars >>
    line_indent = 0 # The indentation to use if we see a section reference.
    # indent is the leading whitespace to be deleted.
    i, indent = g.skip_leading_ws_with_indent(s, 0, self.tab_width)
    << Skip the header line output by tangle >>
    # The top level of the stack represents the root.
    self.push_new_DefNode(self.root_name, indent, 1, 1, True)
    while i < len(s):
        ch = s[i]
        if ch == '\r':
            i += 1 # ignore
        elif ch == '\n':
            << handle the start of a new line >>
        elif g.match(s, i, self.sentinel) and self.is_sentinel_line(s, i):
            << handle a sentinel line  >>
        elif g.match(s, i, self.line_comment) or g.match(s, i, self.verbatim):
            << copy the entire line >>
        elif g.match(s, i, self.comment):
            << copy a multi-line comment >>
        elif g.match(s, i, self.string1) or g.match(s, i, self.string2):
            << copy a string >>
        else:
            self.copy(ch); i += 1
    << end all open sections >>
#@+node:ekr.20210312150753.136: *6* << set the private global matching vars >>
# Set defaults.
self.string1 = "\""
self.string2 = "'"
self.verbatim = None
# Set special cases.
if self.language == "plain":
    self.string1 = self.string2 = None # This is debatable.
# if you're not going to use { } for pascal comments, use
# @comment (* *)
# to specify the alternative
#if self.language == "pascal":
#    self.comment2 = "(*" ; self.comment2_end = "*)"
self.refpart_stack = []
self.select_next_sentinel()
if self.language == "latex": # 3/10/03: Joo-won Jung
    self.string1 = self.string2 = None # This is debatable.
if self.language == "html":
    self.string1 = '"'; self.string2 = None # 12/3/03

#@+node:ekr.20210312150753.137: *6* << Skip the header line output by tangle >>
if self.line_comment or self.comment:
    line = self.line_comment if self.line_comment else self.comment + " Created by Leo from"
    if g.match(s, i, line):
        # Even a block comment will end on the first line.
        i = g.skip_to_end_of_line(s, i)
#@+node:ekr.20210312150753.138: *6* << handle the start of a new line >> (Untangle)
self.copy(ch); i += 1
    # This works because we have one-character newlines.
#
# Set line_indent, used only if we see a section reference.
junk, line_indent = g.skip_leading_ws_with_indent(s, i, c.tab_width)
i = g.skip_leading_ws(s, i, indent, c.tab_width)
    # skip indent leading white space.
#@+node:ekr.20210312150753.139: *6* << handle a sentinel line >>
@
This is the place to eliminate the proper amount of whitespace from
the start of each line. We do this by setting the 'indent' variable to
the leading whitespace of the first _non-blank_ line following the
opening sentinel.

Tangle increases the indentation by one tab if the section reference
is not the first non-whitespace item on the line,so self code must do
the same.
@c

result, junk, kind, name, part, of, end, nl_flag = self.is_sentinel_line_with_data(s, i)
assert(result)
<< terminate the previous part of this section if it exists >>
if kind == start_sentinel_line:
    indent = line_indent
    # Increase line_indent by one tab width if the
    # the section reference does not start the line.
    j = i - 1
    while j >= 0:
        if g.is_nl(s, j):
            break
        elif not g.is_ws(s[j]):
            indent += abs(self.tab_width); break
        j -= 1
    # copy the section reference to the _present_ section,
    # but only if this is the first part of the section.
    if part < 2: self.copy(name)
    # Skip to the first character of the new section definition.
    i = g.skip_to_end_of_line(s, i)
    # Start the new section.
    self.push_new_DefNode(name, indent, part, of, nl_flag)
    self.select_next_sentinel()
else:
    assert(kind == end_sentinel_line)
    # Skip the sentinel line.
    i = g.skip_to_end_of_line(s, i)
    # Skip a newline only if it was added after(!newline)
    if not nl_flag:
        i = g.skip_ws(s, i)
        i = g.skip_nl(s, i)
        i = g.skip_ws(s, i)
        # Copy any whitespace following the (!newline)
        while end and g.is_ws(s[end]):
            self.copy(s[end])
            end += 1
    # Restore the old indentation level.
    if self.def_stack:
        indent = self.def_stack[-1].indent
    self.select_next_sentinel(part_start_flag=False)
#@+node:ekr.20210312150753.140: *7* << terminate the previous part of this section if it exists >>
@
We have just seen a sentinel line. Any kind of sentinel line will
terminate a previous part of the present definition. For end sentinel
lines, the present section name must match the name on the top of the
stack.
@c
if self.def_stack:
    dn = self.def_stack[-1]
    if self.compare_section_names(name, dn.name):
        dn = self.def_stack.pop()
        if dn.code:
            thePart, found = self.ust_lookup(name, dn.part, False, False) # not root, not update
            # Check for incompatible previous definition.
            if found and not self.forgiving_compare(name, dn.part, dn.code, thePart.code):
                self.error("Incompatible definitions of " + name)
            elif not found:
                self.ust_enter(name, dn.part, dn.of, dn.code, dn.nl_flag, False) # not root
    elif kind == end_sentinel_line:
        self.error(f"Missing sentinel line for {name}. found end {dn.name} instead")
#@+node:ekr.20210312150753.141: *6* << copy the entire line >>
j = i; i = g.skip_to_end_of_line(s, i)
self.copy(s[j: i])
#@+node:ekr.20210312150753.142: *6* << copy a multi-line comment >>
assert(self.comment_end)
j = i
i += len(self.comment)
if self.sentinel == self.comment:
    # Scan for the ending delimiter.
    while i < len(s) and not g.match(s, i, self.comment_end):
        i += 1
    if g.match(s, i, self.comment_end):
        i += len(self.comment_end)
    self.copy(s[j: i])
else:
    # Copy line by line, looking for a sentinel within the
    # comment
    while i < len(s):
        if g.match(s, i, self.comment_end):
            i += len(self.comment_end)
            break
        elif g.is_nl(s, i):
            k = g.skip_nl(s, i)
            k = g.skip_ws(s, k)
            if self.is_sentinel_line(s, k):
                break
            else:
                i = k + 1 if k + 1 <= len(s) else len(s)
        else:
            i += 1
self.copy(s[j: i])
#@+node:ekr.20210312150753.143: *6* << copy a string >>
j = i
if self.language == "pascal":
    i = g.skip_pascal_string(s, i)
else:
    i = g.skip_string(s, i)
self.copy(s[j: i])
#@+node:ekr.20210312150753.144: *6* << end all open sections >>
dn = None
while self.def_stack:
    dn = self.def_stack.pop()
    if self.def_stack:
        self.error("Unterminated section: " + dn.name)
if dn:
    # Terminate the root setcion.
    i = len(s)
    if dn.code:
        self.ust_enter(dn.name, dn.part, dn.of, dn.code, dn.nl_flag, is_root_flag=True)
    else:
        self.error("Missing root part")
else:
    self.error("Missing root section")
#@+node:ekr.20210312150753.145: *5* select_next_sentinel & helper
def select_next_sentinel(self, part_start_flag=True):
    '''
    The next sentinel will be either
    (a) a section part reference, using the "before" comment style for that part
    - when there are section references yet to interpolate for this part
    - when we're followed by another part for this section
    (b) an end sentinel using the "after" comment style for the current part
    - when we've exhausted the parts for this section
    or (c) end of file for the root section
    The above requires that the parts in the tst be aware of the section
    interpolations each part will make
    '''
    # keep a "private" copy of the tst table so that it doesn't get
    # corrupted by a subsequent tanglePass1 run
    if not self.delims_table:
        self.delims_table = self.tst
        restore_tst = self.tst
    else:
        restore_tst = self.tst
        self.tst = self.delims_table
    if self.refpart_stack == []:
        # beginning a new file
        section = self.st_lookup(self.root_name)
        assert section.__class__ == TstNode
        assert len(section.parts) == 1
        # references to sections within the part were noted by tanglePass1
        root_part = section.parts[0]
        assert root_part.__class__ == PartNode
        self.push_parts(root_part.reflist())
        # set the delimiters for the root section
        delims = section.parts[0].delims
    else:
        # we've just matched a sentinel
        if part_start_flag:
            part = self.refpart_stack.pop()
            assert part.__class__ == PartNode, (
                "expected type PartNode, got %s" % repr(part.__class__))
            self.push_parts(part.reflist())
        else:
            s = self.refpart_stack.pop()
            assert s.__class__ == TstNode
        if self.refpart_stack:
            delims = self.refpart_stack[-1].delims
        else:
            section = self.st_lookup(self.root_name)
            delims = section.delims
    if delims[0]:
        self.line_comment = delims[0]
        self.sentinel = delims[0]
        self.sentinel_end = False
    else:
        self.line_comment = None
        self.sentinel = delims[1]
        self.sentinel_end = delims[2]
    # don't change multiline comment until after a comment convention transition is finished
    if len(self.refpart_stack) < 2 or (
        self.refpart_stack[-2].delims[1] == self.refpart_stack[-1].delims[1] and
        self.refpart_stack[-2].delims[1] == self.refpart_stack[-1].delims[2]
    ):
        self.comment = delims[1]
        self.comment_end = delims[2]
    self.tst = restore_tst
#@+node:ekr.20210312150753.146: *6* push_parts
def push_parts(self, reflist):
    if not reflist:
        return
    for i in range(-1, -(len(reflist) + 1), -1):
        # push each part start delims for each reference expected
        r = reflist[i]
        # cope with undefined sections
        count = len(r.parts)
        if count > 0:
            # push the section for the end sentinel
            self.refpart_stack.append(r)
            for j in range(-1, -(count + 1), -1):
                self.refpart_stack.append(r.parts[j])
#@+node:ekr.20210312150753.147: *5* update_def (untangle: pass 2)
@
This function handles the actual updating of section definitions in the web.
Only code parts are updated, never doc parts.

During pass 2 of Untangle, skip_body() calls this routine when it discovers the
definition of a section in the outline. We look up the name in the ust. If an
entry exists, we compare the code (the code part of an outline node) with the
code part in the ust. We update the code part if necessary.

We use the forgiving_compare() to compare code parts. It's not possible to
change only trivial whitespace using Untangle because forgiving_compare()
ignores trivial whitespace.
@c
# Major change: 2/23/01: Untangle never updates doc parts.

def update_def(self, name, part_number, head, code, tail, is_root_flag=False):
    # Doc parts are never updated!
    p = self.p; body = p.b
    if not head: head = ""
    if not tail: tail = ""
    if not code: code = ""
    false_ret = head + code + tail, len(head) + len(code), False
    part, found = self.ust_lookup(name, part_number, is_root_flag, update_flag=True)
    if not found:
        return false_ret # Not an error.
    ucode = g.toUnicode(part.code, self.encoding)
    << Remove leading blank lines and comments from ucode >>
    if not ucode:
        return false_ret # Not an error.
    if code and self.forgiving_compare(name, part, code, ucode):
        return false_ret # Not an error.
    # Update the body.
    if not g.unitTesting:
        g.es("***Updating:", p.h)
    ucode = ucode.strip()
    << Add the trailing whitespace of code to ucode. >>
    << Move any @language or @comment from code to ucode >>
    body = head + ucode + tail
    self.update_current_vnode(body)
    return body, len(head) + len(ucode), True
#@+node:ekr.20210312150753.148: *6* << Remove leading blank lines and comments from ucode >>
@
We formerly assumed that any leading comments came from an @doc part, which might
be the case if self.output_doc_flag were true.  For now, we treat leading comments
as "code".

Elsewhere in the code is a comment that "we never update doc parts" when untangling.

Needs to be dealt with.
@c
i = g.skip_blank_lines(ucode, 0)
@
if self.comment and self.comment_end:
    if ucode and g.match(ucode,j,self.comment):
        # Skip to the end of the block comment.
        i = j + len(self.comment)
        i = ucode.find(self.comment_end,i)
        if i == -1: ucode = None # An unreported problem in the user code.
        else:
            i += len(self.comment_end)
            i = g.skip_blank_lines(ucode,i)
elif self.line_comment:
    while ucode and g.match(ucode,j,self.line_comment):
        i = g.skip_line(ucode,i)
        i = g.skip_blank_lines(ucode,i)
        j = g.skip_ws(ucode,i)
 Only the value of ucode matters here.
@c
if ucode: ucode = ucode[i:]
#@+node:ekr.20210312150753.149: *6* << Add the trailing whitespace of code to ucode. >>
code2 = code.rstrip()
trail_ws = code[len(code2):]
ucode = ucode + trail_ws
#@+node:ekr.20210312150753.150: *6* << Move any @language or @comment from code to ucode >>
# split the code into lines, collecting the @language and @comment lines specially
# if @language or @comment are present, they get added at the end
if code[-1] == '\n':
    leading_newline = ''
    trailing_newline = '\n'
else:
    leading_newline = '\n'
    trailing_newline = ''
m = self.RegexpForLanguageOrComment.regex.match(code)
if m.group('language'):
    ucode = ucode + leading_newline
    if m.group('comment') and (m.start('language') < m.start('comment')):
        ucode = ucode + m.group('language') + "\n" + m.group('comment')
    else:
        ucode = ucode + m.group('language')
    ucode = ucode + trailing_newline
else:
    if m.group('comment'):
        ucode = ucode + leading_newline + m.group('comment') + trailing_newline
#@+node:ekr.20210312150753.151: *5* update_current_vnode
def update_current_vnode(self, s):
    """Called from within the Untangle logic to update the body text of self.p."""
    c = self.c; p = self.p
    assert(self.p)
    c.setBodyString(p, s)
    c.setChanged()
    p.setDirty()
    p.setMarked()
    # 2010/02/02: was update_after_icons_changed.
    c.redraw_after_icons_changed()
#@+node:ekr.20210312150753.152: *4* utility methods
@ These utilities deal with tangle ivars, so they should be methods.
#@+node:ekr.20210312150753.153: *5* compare_section_names
# Compares section names or root names.
# Arbitrary text may follow the section name on the same line.

def compare_section_names(self, s1, s2):

    if g.match(s1, 0, "<<") or g.match(s1, 0, "@<"):
        # Use a forgiving compare of the two section names.
        delim = ">>"
        i1 = i2 = 0
        while i1 < len(s1) and i2 < len(s2):
            ch1 = s1[i1]; ch2 = s2[i2]
            if g.is_ws(ch1) and g.is_ws(ch2):
                i1 = g.skip_ws(s1, i1)
                i2 = g.skip_ws(s2, i2)
            elif g.match(s1, i1, delim) and g.match(s2, i2, delim):
                return True
            elif ch1.lower() == ch2.lower():
                i1 += 1; i2 += 1
            else: return False
        return False
    # A root name.
    return s1 == s2
#@+node:ekr.20210312150753.154: *5* copy
def copy(self, s):
    assert self.def_stack
    # dn = self.def_stack[-1] # Add the code at the top of the stack.
    # dn.code += s
    self.def_stack[-1].code += s
        # Add the code at the top of the stack.
        # pyflakes has trouble with the commented-out code.
#@+node:ekr.20210312150753.155: *5* error, pathError, warning
def error(self, s):
    self.errors += 1
    g.es_error(g.translateString(s))

def pathError(self, s):
    if not self.path_warning_given:
        self.path_warning_given = True
        self.error(s)

def warning(self, s):
    g.es_error(g.translateString(s))
#@+node:ekr.20210312150753.156: *5* is_end_of_directive
# This function returns True if we are at the end of preprocessor directive.

def is_end_of_directive(self, s, i):
    return g.is_nl(s, i) and not self.is_escaped(s, i)
#@+node:ekr.20210312150753.157: *5* is_end_of_string
def is_end_of_string(self, s, i, delim):
    return i < len(s) and s[i] == delim and not self.is_escaped(s, i)
#@+node:ekr.20210312150753.158: *5* is_escaped
# This function returns True if the s[i] is preceded by an odd number of back slashes.

def is_escaped(self, s, i):
    back_slashes = 0; i -= 1
    while i >= 0 and s[i] == '\\':
        back_slashes += 1
        i -= 1
    return (back_slashes & 1) == 1
#@+node:ekr.20210312150753.159: *5* is_section_name
def is_section_name(self, s, i):
    kind = bad_section_name; end = -1
    if g.match(s, i, "<<"):
        i, kind, end = self.skip_section_name(s, i)
    return i, kind, end
#@+node:ekr.20210312150753.160: *5* is_sentinel_line & is_sentinel_line_with_data
@
This function returns True if i points to a line a sentinel line of
one of the following forms:

start_sentinel <<section name>> end_sentinel
start_sentinel <<section name>> (n of m) end_sentinel
start_sentinel -- end -- <<section name>> end_sentinel
start_sentinel -- end -- <<section name>> (n of m) end_sentinel

start_sentinel: the string that signals the start of sentinel lines\
end_sentinel:   the string that signals the endof sentinel lines.

end_sentinel may be None,indicating that sentinel lines end with a newline.

Any of these forms may end with (!newline), indicating that the
section reference was not followed by a newline in the orignal text.
We set nl_flag to False if such a string is seen. The name argument
contains the section name.

The valid values of kind param are:

non_sentinel_line,   # not a sentinel line.
start_sentinel_line, #   /// <section name> or /// <section name>(n of m)
end_sentinel_line  //  /// -- end -- <section name> or /// -- end -- <section name>(n of m).
@c

def is_sentinel_line(self, s, i):
    result, i, kind, name, part, of, end, nl_flag = self.is_sentinel_line_with_data(s, i)
    return result

def is_sentinel_line_with_data(self, s, i):
    start_sentinel = self.sentinel
    end_sentinel = self.sentinel_end
    << Initialize the return values >>
    << Make sure the line starts with start_sentinel >>
    << Set end_flag if we have -- end -- >>
    << Make sure we have a section reference >>
    << Set part and of if they exist >>
    << Set nl_flag to False if !newline exists >>
    << Make sure the line ends with end_sentinel >>
    kind = end_sentinel_line if end_flag else start_sentinel_line
    return True, i, kind, name, part, of, end, nl_flag
#@+node:ekr.20210312150753.161: *6* << Initialize the return values  >>
name = end = None
part = of = 1
kind = non_sentinel_line
nl_flag = True
false_data = (False, i, kind, name, part, of, end, nl_flag)
#@+node:ekr.20210312150753.162: *6* << Make sure the line starts with start_sentinel >>
if g.is_nl(s, i): i = g.skip_nl(s, i)
i = g.skip_ws(s, i)
# 4/18/00: We now require an exact match of the sentinel.
if g.match(s, i, start_sentinel):
    i += len(start_sentinel)
else:
    return false_data
#@+node:ekr.20210312150753.163: *6* << Set end_flag if we have -- end -- >>
# If i points to "-- end --", this code skips it and sets end_flag.
end_flag = False
i = g.skip_ws(s, i)
if g.match(s, i, "--"):
    while i < len(s) and s[i] == '-':
        i += 1
    i = g.skip_ws(s, i)
    if not g.match(s, i, "end"):
        return false_data # Not a valid sentinel line.
    i += 3; i = g.skip_ws(s, i)
    if not g.match(s, i, "--"):
        return false_data # Not a valid sentinel line.
    while i < len(s) and s[i] == '-':
        i += 1
    end_flag = True
#@+node:ekr.20210312150753.164: *6* << Make sure we have a section reference >>
i = g.skip_ws(s, i)
if g.match(s, i, "<<"):
    j = i; i, kind, end = self.skip_section_name(s, i)
    if kind != section_ref:
        return false_data
    name = s[j: i]
else:
    return false_data
#@+node:ekr.20210312150753.165: *6* << Set part and of if they exist >>
# This code handles (m of n), if it exists.
i = g.skip_ws(s, i)
if g.match(s, i, '('):
    j = i
    i += 1; i = g.skip_ws(s, i)
    i, part = self.scan_short_val(s, i)
    if part == -1:
        i = j # back out of the scanning for the number.
        part = 1
    else:
        i = g.skip_ws(s, i)
        if not g.match(s, i, "of"):
            return false_data
        i += 2; i = g.skip_ws(s, i)
        i, of = self.scan_short_val(s, i)
        if of == -1:
            return false_data
        i = g.skip_ws(s, i)
        if g.match(s, i, ')'):
            i += 1 # Skip the paren and do _not_ return.
        else:
            return false_data
#@+node:ekr.20210312150753.166: *6* << Set nl_flag to false if !newline exists >>
line = "(!newline)"
i = g.skip_ws(s, i)
if g.match(s, i, line):
    i += len(line)
    nl_flag = False
#@+node:ekr.20210312150753.167: *6* << Make sure the line ends with end_sentinel >>
i = g.skip_ws(s, i)
if end_sentinel:
    # Make sure the line ends with the end sentinel.
    if g.match(s, i, end_sentinel):
        i += len(end_sentinel)
    else:
        return false_data
end = i # Show the start of the whitespace.
i = g.skip_ws(s, i)
if i < len(s) and not g.is_nl(s, i):
    return false_data
#@+node:ekr.20210312150753.168: *5* parent_language_comment_settings
# side effect: sets the values within lang_dict
# *might* lower case c.target_language

def parent_language_comment_settings(self, p, lang_dict):
    c = self.c
    if p.hasParent():
        p1 = p.parent()
        for s in (p1.b, p1.h):
            m = self.RegexpForLanguageOrComment.regex.match(s)
            if not lang_dict['delims']:
                if m.group('language'):
                    lang, d1, d2, d3 = g.set_language(m.group('language'), 0)
                    lang_dict['language'] = lang
                    lang_dict['delims'] = (d1, d2, d3)
                    if m.group('comment') and (m.start('comment') > m.start('language')):
                        lang_dict['delims'] = g.set_delims_from_string(m.group('comment'))
                    break
                if m.group('comment'):
                    lang_dict['delims'] = g.set_delims_from_string(m.group('comment'))
            elif not lang_dict['language']:
                # delims are already set, only set language
                if m.group('language'):
                    lang, d1, d2, d3 = g.set_language(m.group('language'), 0)
                    lang.dict['language'] = lang
                    break
        if not lang_dict['language']:
            self.parent_language_comment_settings(p1, lang_dict)
    else:
        if not lang_dict['language']:
            if c.target_language:
                c.target_language = c.target_language.lower()
                lang, d1, d2, d3 = g.set_language(c.target_language, 0)
                lang_dict['language'] = lang
                if not lang_dict['delims']:
                    lang_dict['delims'] = (d1, d2, d3)
#@+node:ekr.20210312150753.169: *5* push_new_DefNode
# This function pushes a new DefNode on the top of the section stack.

def push_new_DefNode(self, name, indent, part, of, nl_flag):

    node = DefNode(name, indent, part, of, nl_flag, None)
    self.def_stack.append(node)
#@+node:ekr.20210312150753.170: *5* refpart_stack_dump
def refpart_stack_dump(self):
    s = "top of stack:"
    for i in range(-1, -(len(self.refpart_stack) + 1), -1):
        if self.refpart_stack[i].__class__ == PartNode:
            s += ("\nnode: " + self.refpart_stack[i].name +
                  " delims: " + repr(self.refpart_stack[i].delims))
        elif self.refpart_stack[i].__class__ == TstNode:
            s += ("\nsection: " + self.refpart_stack[i].name +
            " delims: " + repr(self.refpart_stack[i].delims))
        else:
            s += "\nINVALID ENTRY of type " + repr(self.refpart_stack[i].__class__)
    s += "\nbottom of stack.\n"
    return s
#@+node:ekr.20210312150753.171: *5* scan_short_val
# This function scans a positive integer.
# returns (i,val), where val == -1 if there is an error.

def scan_short_val(self, s, i):
    if i >= len(s) or not s[i].isdigit():
        return i, -1
    j = i
    while i < len(s) and s[i].isdigit():
        i += 1
    val = int(s[j: i])
    return i, val
#@+node:ekr.20210312150753.172: *5* setRootFromHeadline
def setRootFromHeadline(self, p):
    s = p.h
    if s[0: 5] == "@root":
        i, self.start_mode = g.scanAtRootOptions(s, 0)
        i = g.skip_ws(s, i)
        if i < len(s): # Non-empty file name.
            # self.root_name must be set later by token_type().
            self.root = s[i:]
            # implement headline @root (but create unit tests first):
            # arguments: name, is_code, is_doc
            # st_enter_root_name(self.root, False, False)
#@+node:ekr.20210312150753.173: *5* setRootFromText
@ This code skips the file name used in @root directives.

File names may be enclosed in < and > characters, or in double quotes.
If a file name is not enclosed be these delimiters it continues until
the next newline.
@c

def setRootFromText(self, s, report_errors=True):

    self.root_name = None
    i, self.start_mode = g.scanAtRootOptions(s, 0)
    i = g.skip_ws(s, i)
    if i >= len(s): return i
    # Allow <> or "" as delimiters, or a bare file name.
    if s[i] == '"':
        i += 1; delim = '"'
    elif s[i] == '<':
        i += 1; delim = '>'
    else: delim = '\n'
    root1 = i # The name does not include the delimiter.
    while i < len(s) and s[i] != delim and not g.is_nl(s, i):
        i += 1
    root2 = i
    if delim != '\n' and not g.match(s, i, delim):
        if report_errors:
            g.scanError("bad filename in @root " + s[: i])
    else:
        self.root_name = s[root1: root2].strip()
    return i
#@+node:ekr.20210312150753.174: *5* skip_section_name
@
This function skips past a section name that starts with < < and might
end with > > or > > =. The entire section name must appear on the same
line.

Note: this code no longer supports extended noweb mode.

Returns (i, kind, end),
    end indicates the end of the section name itself (not counting the =).
    kind is one of:
        bad_section_name: "no matching ">>" or ">>"  This is _not_ a user error!
        section_ref: < < name > >
        section_def: < < name > > =
        at_root:     < < * > > =
@c

def skip_section_name(self, s, i):
    assert(g.match(s, i, "<<"))
    i += 2
    j = i # Return this value if no section name found.
    kind = bad_section_name; end = -1; empty_name = True
    # Scan for the end of the section name.
    while i < len(s) and not g.is_nl(s, i):
        if g.match(s, i, ">>="):
            i += 3; end = i - 1; kind = section_def; break
        elif g.match(s, i, ">>"):
            i += 2; end = i; kind = section_ref; break
        elif g.is_ws_or_nl(s, i):
            i += 1
        elif empty_name and s[i] == '*':
            empty_name = False
            i = g.skip_ws(s, i + 1) # skip the '*'
            if g.match(s, i, ">>="):
                i += 3; end = i - 1; kind = at_root; break
        else:
            i += 1; empty_name = False
    if empty_name:
        kind = bad_section_name
    if kind == bad_section_name:
        i = j
    return i, kind, end
#@+node:ekr.20210312150753.175: *5* standardize_name
def standardize_name(self, name):
    """
    Removes leading and trailing brackets,
    converts white space to a single blank and
    converts to lower case.
    """
    # Convert to lowercase.
    # Convert whitespace to a single space.
    name = name.lower().replace('\t', ' ').replace('  ', ' ')
    # Remove leading '<'
    i = 0; n = len(name)
    while i < n and name[i] == '<':
        i += 1
    j = i
    # Find the first '>'
    while i < n and name[i] != '>':
        i += 1
    name = name[j: i].strip()
    return name
#@+node:ekr.20210312150753.176: *5* tangle.scanAllDirectives
def scanAllDirectives(self, p):
    """Scan VNode p and p's ancestors looking for directives,
    setting corresponding tangle ivars and globals.
    """
    c = self.c
    self.init_directive_ivars()
    if not p:
        p = c.p
    if p:
        s = p.b
        << Collect @first attributes >>
    table = (
        ('encoding', self.encoding, g.scanAtEncodingDirectives),
        ('lineending', None, g.scanAtLineendingDirectives),
        ('pagewidth', c.page_width, g.scanAtPagewidthDirectives),
        ('path', None, c.scanAtPathDirectives),
        ('tabwidth', c.tab_width, g.scanAtTabwidthDirectives),
    )
    # Set d by scanning all directives.
    aList = g.get_directives_dict_list(p)
    d = {}
    for key, default, func in table:
        val = func(aList)
        d[key] = default if val is None else val
    lang_dict = {'language': None, 'delims': None}
    self.parent_language_comment_settings(p, lang_dict)
    # Post process.
    lineending = d.get('lineending')
    if lineending:
        self.output_newline = lineending
    self.encoding = d.get('encoding')
    self.language = lang_dict.get('language')
    self.init_delims = lang_dict.get('delims')
    self.page_width = d.get('pagewidth')
    self.tangle_directory = d.get('path')
    self.tab_width = d.get('tabwidth')
    # Handle the print-mode directives.
    self.print_mode = None
    for d in aList:
        for key in ('verbose', 'terse', 'quiet', 'silent'):
            if d.get(key) is not None:
                self.print_mode = key; break
        if self.print_mode: break
    if not self.print_mode: self.print_mode = 'verbose'
    # For unit testing.
    return {
        "encoding": self.encoding,
        "language": self.language,
        "lineending": self.output_newline,
        "pagewidth": self.page_width,
        "path": self.tangle_directory,
        "tabwidth": self.tab_width,
    }
#@+node:ekr.20210312150753.177: *6* << Collect @first attributes >>
@
Stephen P. Schaefer 9/13/2002: Add support for @first. Unlike other
root attributes, does *NOT* inherit from parent nodes.
@c
tag = "@first"
sizeString = len(s) # DTHEIN 13-OCT-2002: use to detect end-of-string
i = 0
while 1:
    # DTHEIN 13-OCT-2002: directives must start at beginning of a line
    if not g.match_word(s, i, tag):
        i = g.skip_line(s, i)
    else:
        i = i + len(tag)
        j = i = g.skip_ws(s, i)
        i = g.skip_to_end_of_line(s, i)
        if i > j:
            self.first_lines += s[j: i] + '\n'
        i = g.skip_nl(s, i)
    if i >= sizeString: # DTHEIN 13-OCT-2002: get out when end of string reached
        break
#@+node:ekr.20210312150753.178: *5* tangle.update_file_if_changed
def update_file_if_changed(self, c, file_name, temp_name):
    """
    A helper for compares two files.

    If they are different, we replace file_name with temp_name.
    Otherwise, we just delete temp_name. Both files should be closed.
    """
    import filecmp
    if g.os_path_exists(file_name):
        if filecmp.cmp(temp_name, file_name):
            kind = 'unchanged'
            ok = g.utils_remove(temp_name)
        else:
            kind = '***updating'
            mode = g.utils_stat(file_name)
            ok = g.utils_rename(c, temp_name, file_name, mode)
    else:
        kind = 'creating'
        head, tail = g.os_path_split(file_name)
        ok = True
        if (
            head and c and c.config and
            c.config.create_nonexistent_directories
        ):
            theDir = c.expand_path_expression(head)
            if theDir:
                ok = g.makeAllNonExistentDirectories(theDir)
        if ok:
            ok = g.utils_rename(c, temp_name, file_name)
    if ok:
        g.es('', f'{kind:12}: {file_name}')
    else:
        g.error("rename failed: no file created!")
        g.es('', file_name, " may be read-only or in use")
#@+node:ekr.20210312150753.179: *5* token_type
def token_type(self, s, i, report_errors=True):
    """This method returns a code indicating the apparent kind of token at the position i.

    The caller must determine whether section definiton tokens are valid.

    returns (kind, end) and sets global root_name using setRootFromText().
    end is only valid for kind in (section_ref, section_def, at_root)."""
    kind = plain_line; end = -1
    << set token_type in noweb mode >>
    if kind == at_other:
        << set kind for directive >>
    return kind, end
#@+node:ekr.20210312150753.180: *6* << set token_type in noweb mode >>
if g.match(s, i, "<<"):
    i, kind, end = self.skip_section_name(s, i)
    if kind == bad_section_name:
        kind = plain_line # not an error.
    elif kind == at_root:
        assert(self.head_root is None)
        if self.head_root:
            self.setRootFromText(self.head_root, report_errors)
        else:
            kind = bad_section_name # The warning has been given.
elif g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@\n"):
    # 10/30/02: Only @doc starts a noweb doc part in raw cweb mode.
    kind = plain_line if self.raw_cweb_flag else at_doc
elif g.match(s, i, "@@"): kind = at_at
elif i < len(s) and s[i] == '@': kind = at_other
else: kind = plain_line
#@+node:ekr.20210312150753.181: *6* << set kind for directive >>
# This code will return at_other for any directive other than those listed.
if g.match_word(s, i, "@c"):
    # 10/30/02: Only @code starts a code section in raw cweb mode.
    kind = plain_line if self.raw_cweb_flag else at_code
else:
    for name, theType in [
        ("@chapter", at_chapter),
        ("@code", at_code),
        ("@doc", at_doc),
        ("@root", at_root),
        ("@section", at_section)
    ]:
        if g.match_word(s, i, name):
            kind = theType; break
if self.raw_cweb_flag and kind == at_other:
    # 10/30/02: Everything else is plain text in raw cweb mode.
    kind = plain_line
if kind == at_root:
    end = self.setRootFromText(s[i:], report_errors)
=======

    d = {}
    
    def __getattr__(self, name):
        return self.d.get(name, 0)
        
    def __setattr__(self, name, val):
        self.d[name] = val
        
    def report(self):
        if self.d:
            n = max([len(key) for key in self.d])
            for key, val in sorted(self.d.items()):
                print('%*s: %s' % (n, key, val))
        else:
            print('no stats')
#@+node:ekr.20210312150753.182: *3* top-level test functions
#@+node:ekr.20210312150753.183: *4* testShowData (leoCheck.py)
def test(c, files):
    r"""
    A stand-alone version of @button show-data.  Call as follows:

        import leo.core.leoCheck as leoCheck
        files = (
            [
                # r'c:\leo.repo\leo-editor\leo\core\leoNodes.py',
            ] or
            leoCheck.ProjectUtils().project_files('leo')
        )
        leoCheck.test(files)
    """
    # pylint: disable=import-self
    import leo.core.leoCheck as leoCheck
    leoCheck.ShowData(c=c).run(files)
#@+node:ekr.20210312150753.184: *3* checkConventions (checkerCommands.py)
@g.command('check-conventions')
@g.command('cc')
def checkConventions(event):
    """Experimental script to test Leo's convensions."""
    c = event.get('c')
    if c:
        if c.changed: c.save()
        import importlib
        import leo.core.leoCheck as leoCheck
        importlib.reload(leoCheck)
        leoCheck.ConventionChecker(c).check()
#@+node:ekr.20200209055601.1: ** Test stuff
#@+node:ekr.20200505050955.1: *3* ----- mvc-prototype branch
@language rest
@wrap

- checkout the mvc-prototype branch

- open leo/core/LeoPyRef.leo in Leo and save this document as leo/core/LeoPyRef.db
  This should provide a large outline for the prototype to play with.

- python leo/extensions/myleoqt.py leo\core\leoPy.db
#@+node:vitalije.20200502083732.1: *4* @@@file ../extensions/myleoqt.py
#@+node:vitalije.20200502091628.1: *4* <<imports>>
from collections import defaultdict
import sys
import os
import pickle
from PyQt5 import QtCore, QtGui, QtWidgets
assert QtGui
import sqlite3
import time
import timeit
import unittest
from hypothesis.strategies import lists, integers, sampled_from, data
from hypothesis import given, settings
from datetime import timedelta
import random

# Set the path before importing Leo files.
LEO_INSTALLED_AT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
if LEO_INSTALLED_AT not in sys.path:
    sys.path.append(LEO_INSTALLED_AT)
LEO_ICONS_DIR = os.path.join(LEO_INSTALLED_AT, 'leo', 'themes', 'light', 'Icons')
try:
    import leo.core.leoNodes as leoNodes
    import leo.core.leoGlobals as g
except ImportError:
    print('===== import error!')
g.app = g.bunch(nodeIndices=leoNodes.NodeIndices('vitalije'))

Q = QtCore.Qt
#@+node:vitalije.20200502103535.1: *4* draw_tree
def draw_tree(tree, root, icons):
    def additem(par, parItem):
        for ch in par.children:
            item = QtWidgets.QTreeWidgetItem()
            item.setFlags(item.flags() |
                          Q.ItemIsEditable |
                          item.DontShowIndicatorWhenChildless)
            parItem.addChild(item)
            item.setText(0, ch.h)
            item.setData(0, 1024, ch)
            item.setExpanded(ch.isExpanded())
            item.setIcon(0, icons[ch.computeIcon()])
            additem(ch, item)
    ritem = tree.invisibleRootItem()
    ritem.setData(0, 1024, root)
    additem(root, ritem)
#@+node:vitalije.20200508091419.1: *4* execute_script
def execute_script(self):
    c = self.c
    t = self.tree
    root = t.invisibleRootItem()
    curr = t.currentItem()
    currpath = path_to_item(root, curr)

    # now we need to replace current top level items
    # with the their clones so that we can preserve
    # the original ones for undo
    t.blockSignals(True)
    n = root.childCount()
    olditems = [root.takeChild(0) for x in range(n)]
    for item in olditems:
        root.addChild(item.clone())

    # let's find current item in the new tree
    ncurr = item_from_path(root, currpath)
    t.setCurrentItem(ncurr)
    t.blockSignals(False)

    # store old v-nodes data
    old_vs = set((x, x.h, x.b) for x in c.fileCommands.gnxDict.values())

    # new values for the redo operation
    # if the script finishes without exception
    # these values will be updated after the script finishes
    # for now they are the same as the old values
    newitems = olditems
    new_vs = old_vs
    newcurr = curr

    # some more data for the locals in script
    p = QtPosition(ncurr)
    v = p.v
    script = v.b

    @others

    try:
        exec(script, dict(c=c, v=v, p=p, tree=t, root=root))
        # update new values and redraw the tree
        nset = set((x, x.h, x.b) for x in c.fileCommands.gnxDict.values())
        new_vs =  nset - old_vs
        old_vs = old_vs - nset
        item = t.currentItem()
        newv = item.data(0, 1024)
        npath = path_to_item(root, item)
        self.redraw()

        newcurr = item_from_path(root, npath)
        # we want to keep the same node selected if possible
        # but script may delete previously selected node or
        # some other node so their position won't match
        if not newcurr or newcurr.data(0, 1024) != newv:
            # let's try to find item that points to the newv
            for item in iter_all_v_items(root, newv):
                newcurr = item
                break

        # remember the newitems for redo
        n2 = root.childCount()
        newitems = [root.child(i) for i in range(n2)]

        # select node if we found one to select
        if newcurr:
            t.setCurrentItem(newcurr)

    except KeyboardInterrupt:
        # sometimes script got into an infinite loop
        # or work too long
        undoexec()
        return
    except:
        g.es_print_exception()
        undoexec()
        return
    # script terminated successfully let's add undo/redo pair
    self.c.addUndo(undoexec, redoexec)
#@+node:vitalije.20200508104717.1: *5* undoexec
def undoexec():
    for v, h, b in old_vs:
        v.b = b
        v.h = h
    t.blockSignals(True)
    root.takeChildren()
    for item in olditems:
        root.addChild(item)
    reconnect_vnodes(root)
    t.blockSignals(False)
    t.setCurrentItem(curr)
#@+node:vitalije.20200508104721.1: *5* redoexec
def redoexec():
    for v, h, b in new_vs:
        v.b = b
        v.h = h
    t.blockSignals(True)
    root.takeChildren()
    for item in newitems:
        root.addChild(item)
    reconnect_vnodes(root)
    t.blockSignals(False)
    t.setCurrentItem(newcurr)
#@+node:vitalije.20200507182943.1: *4* test_select_and_commnads
@settings(max_examples=50, deadline=timedelta(seconds=4))
@given(data())
def test_select_and_commnads(data):
    app = demo2_app()

    index = data.draw(integers(min_value=0, max_value=10000))
    names = data.draw(lists(sampled_from(method_names), min_size=5,
                             max_size=100))

    select_item_at_index(app, index)
    a1, b1 = outline_shapes(app)
    for name in names:
        meth = getattr(app, name)
        meth()
        app.processEvents() #
        if method_names.index(name) > 3:
            # possible change in the outline
            a2, b2 = outline_shapes(app)
            assert a2 == b2
            if a1 != a2:
                # the change is real - let's undo it
                app.undo()
                a, b = outline_shapes(app)
                assert a == a1
                assert b == b1
                app.redo()
            a1, b1 = a2, b2
#@+node:ekr.20210429050530.1: *3* @@button test-ConvertAtRoot (obsolete)
g.cls()
import importlib
import leo.commands.editFileCommands as editFileCommands

if c.isChanged():
    c.save()

importlib.reload(editFileCommands)

path = r'c:\test\atRoot.leo'
hidden_c = g.createHiddenCommander(path)
if hidden_c:
    editFileCommands.ConvertAtRoot().convert_file(hidden_c)
    print('done')
else:
    print('not found', path)
#@+node:ekr.20200219073857.1: *3* @@suite run all doctests
tm = c.testManager
path = g.os_path_join(g.app.loadDir,"..","core")
exclude = ['leoIPython.py',]
    # leoIPython.py will give an annoying error if IPython can't be imported.
modules = tm.importAllModulesInPath(path,exclude=exclude)
suite   = tm.createUnitTestsFromDoctests(modules)
#@+node:ekr.20210326183053.1: *3* @@test rst.initAtAutoWrite
g.cls()
rst = c.rstCommands
rst.initAtAutoWrite(p)

# Ensure we are actually testing the default logic.
d = p.v.u.get('rst-import',{})
underlines = d.get('underline_characters')
assert underlines is None,'fail 1: %s' % repr(underlines)
assert d == {},'fail 2: %s' % repr(d)
# Now test the logic.
assert rst.underlines2 == '','fail 3: %s' % repr(rst.underlines2)
assert rst.underlines1 == '=+*^~"\'`-:><_', 'fail4 %s' % repr(rst.underlines1)
# assert rst.atAutoWriteUnderlines == '=+*^~"\'`-:><_', 'fail 5: %s' % (
#    repr(rst.atAutoWriteUnderlines))
print('done')
#@-all
#@@nosearch
#@-leo
