#@+leo-ver=5-thin
#@+node:ekr.20170302123956.1: * @file ../doc/leoAttic.txt
# This is Leo's final resting place for dead code.
# Much easier to access than a git repo.

#@@language python
#@@killbeautify
#@+all
#@+node:ekr.20190412154439.1: **  Abandoned projects
#@+node:ekr.20170203080350.1: *3* Abandoned #396: Show images in Leo's body pane
https://github.com/leo-editor/leo-editor/issues/396

#@+node:ekr.20170302151109.1: *4* ** Notes
@language rest
@wrap

Unicode 'object replacement character': u+FFFC

QTextDocument may be helpful: http://doc.qt.io/qt-5/qtextdocument.html
See QTextDocument.MetaInformation: http://doc.qt.io/qt-5/qtextdocument.html#MetaInformation-enum

http://stackoverflow.com/questions/3254652/
several-ways-of-placing-an-image-in-a-qtextedit

http://doc.qt.io/qt-5/qtextdocument.html#resource

QVariant QTextDocument::resource(int type, const QUrl &name) const

Returns data of the specified type from the resource with the given name.

This function is called by the rich text engine to request data that isn't directly stored by QTextDocument, but still associated with it. For example, images are referenced indirectly by the name attribute of a QTextImageFormat object.

Resources are cached internally in the document. If a resource can not be found in the cache, loadResource is called to try to load the resource. loadResource should then use addResource to add the resource to the cache.
#@+node:ekr.20170203105538.1: *4* Test inserting picture (new)
# https://github.com/leo-editor/leo-editor/issues/396
g.cls()
images = [i for i in range(len(p.b)) if ord(p.b[i]) > 128]
if images:
    print('images at', images)
else:
    table = (
        'application-x-leo-outline.png',
        'LeoDoc.ico',
    )    
    for image in table:
        path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
        assert g.os_path_exists(path), repr(path)
        c.frame.body.wrapper.setInsertPoint(len(p.b))
        if 0:
            format = QtGui.QTextImageFormat()
            format.setName(path)
            cursor = cursor = body.widget.textCursor()
            cursor.insertImage(format)
        c.frame.body.widget.insertHtml('<img src="%s">' % path)
            # style="width:40px;height:80px;"
    for i, ch in enumerate(p.b):
        if ord(ch) > 128: print('new', i, ord(ch))
#
#￼￼
#@+node:ekr.20170204110006.1: *4* test3 insert an image
g.cls()
from leo.core.leoQt import QtGui
body = c.frame.body
table = ('box01.bmp','box02.bmp','box03.bmp',)
d = g.app.permanentScriptDict
images = d.get('images', [])
for image in table:
    path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
    assert g.os_path_exists(path), repr(path)
    image = QtGui.QImage(path)
    images.append(image)
    body.wrapper.setInsertPoint(len(p.b))
    cursor = body.widget.textCursor()
    cursor.insertImage(image)
for i, ch in enumerate(p.b):
    if ord(ch) > 128: print('new', i, ord(ch))
d ['images'] = images
#
#
#@+node:ekr.20170204140521.1: *4* clear g.app.permanentScriptDict
g.printDict(g.app.permanentScriptDict)
g.app.permanentScriptDict = {}
#@+node:ekr.20170204135338.1: *4* @@button show-images
from leo.core.leoQt import QtCore
d = g.app.permanentScriptDict
name_index = d.get('name_index', 0)
names = ['leo_image%s' % (i) for i in range(name_index)]
# print('image names', names)
widget = c.frame.body.widget
doc = widget.document()
for i, name in enumerate(names):
    image = doc.resource(doc.ImageResource, QtCore.QUrl(name))
    print(name, image)
#@+node:ekr.20170204105958.1: *4* test2 insert an image
g.cls()
from leo.core.leoQt import QtGui
body = c.frame.body
table = (
    'application-x-leo-outline.png',
    'LeoDoc.ico',
)
g.app.permanentScriptDict = {}
d = g.app.permanentScriptDict
images = d.get('images', [])
cursors = d.get('cursors', [])
name_index = d.get('name_index', 0)
for image in table:
    path = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons', image)
    assert g.os_path_exists(path), repr(path)
    image = QtGui.QImage(path)
    print(image.format().name())
    images.append(image)
    body.wrapper.setInsertPoint(len(p.b))
    cursor = body.widget.textCursor()
    #name = 'leo_image%s' % name_index
    #name_index += 1
    cursor.insertImage(image)
    cursors.append(cursor)
for i, ch in enumerate(p.b):
    if ord(ch) > 128: print('new', i, ord(ch))
d ['images'] = images
d ['cursors'] = cursors
d ['name_index'] = name_index
g.app.permanentScriptDict = d
g.printDict(d)
#
#￼￼￼
#￼
#@+node:ekr.20190910020834.1: *3* black
#@+node:ekr.20190910023018.1: *4* imports
try:
    # pylint: disable=import-error
        # We can't assume the user has this.
    import black
except Exception:
    black = None
#@+node:ekr.20150531042746.1: *4* munging leo directives
#@+node:ekr.20150529084212.1: *5* comment_leo_lines (leoBeautifier.py)
def comment_leo_lines(p):
    '''Replace lines with Leonine syntax with special comments.'''
    # Choose the comment string so it appears nowhere in s.
    s0 = p.b
    n = 5
    while s0.find('#' + ('!' * n)) > -1:
        n += 1
    comment = '#' + ('!' * n)
    # Create a dict of directives.
    d = {}
    for z in g.globalDirectiveList:
        d[z] = True
    # Convert all Leonine lines to special comments.
    i, lines, result = 0, g.splitLines(s0), []
    while i < len(lines):
        progress = i
        s = lines[i]
        # Comment out any containing a section reference.
        j = s.find('<<')
        k = s.find('>>') if j > -1 else -1
        if -1 < j < k:
            result.append(comment + s)
            # Generate a properly-indented pass line.
            j2 = g.skip_ws(s, 0)
            result.append('%spass\n' % (' ' * j2))
        elif s.lstrip().startswith('@'):
            # Comment out all other Leonine constructs.
            if starts_doc_part(s):
                # Comment the entire doc part, until @c or @code.
                result.append(comment + s)
                i += 1
                while i < len(lines):
                    s = lines[i]
                    result.append(comment + s)
                    i += 1
                    if ends_doc_part(s):
                        break
            else:
                j = g.skip_ws(s, 0)
                assert s[j] == '@'
                j += 1
                k = g.skip_id(s, j, chars='-')
                if k > j:
                    word = s[j: k]
                    if word == 'others':
                        # Remember the original @others line.
                        result.append(comment + s)
                        # Generate a properly-indented pass line.
                        result.append('%spass\n' % (' ' * (j - 1)))
                    else:
                        # Comment only Leo directives, not decorators.
                        result.append(comment + s if word in d else s)
                else:
                    result.append(s)
        else:
            # A plain line.
            result.append(s)
        if i == progress:
            i += 1
    return comment, ''.join(result)
#@+node:ekr.20150531042830.1: *5* starts_doc_part & ends_doc_part
def starts_doc_part(s):
    '''Return True if s word matches @ or @doc.'''
    for delim in ('@\n', '@doc\n', '@ ', '@doc '):
        if s.startswith(delim):
            return True
    return False

def ends_doc_part(s):
    '''Return True if s word matches @c or @code.'''
    for delim in ('@c\n', '@code\n', '@c ', '@code '):
        if s.startswith(delim):
            return True
    return False
#@+node:ekr.20150529095117.1: *5* uncomment_leo_lines
def uncomment_leo_lines(comment, p, s0):
    '''Reverse the effect of comment_leo_lines.'''
    lines = g.splitLines(s0)
    i, result = 0, []
    while i < len(lines):
        progress = i
        s = lines[i]
        i += 1
        if s.find(comment) == -1:
            # A regular line.
            result.append(s)
        else:
            # One or more special lines.
            i = uncomment_special_lines(comment, i, lines, p, result, s)
        assert progress < i
    return ''.join(result).rstrip() + '\n'
#@+node:ekr.20150531041720.1: *5* uncomment_special_line & helpers
def uncomment_special_lines(comment, i, lines, p, result, s):
    '''
    s is a line containing the comment delim.
    i points at the *next* line.
    Handle one or more lines, appending stripped lines to result.
    '''
    s = s.lstrip().lstrip(comment)
    if starts_doc_part(s):
        result.append(s)
        while i < len(lines):
            s = lines[i].lstrip().lstrip(comment)
            i += 1
            result.append(s)
            if ends_doc_part(s):
                break
        return i
    j = s.find('<<')
    k = s.find('>>') if j > -1 else -1
    if -1 < j < k or s.find('@others') > -1:
        # A section reference line or an @others line.
        # Such lines are followed by a pass line.
        # The beautifier may insert blank lines before the pass line.
        kind = 'section ref' if -1 < j < k else '@others'
        # Restore the original line, including leading whitespace.
        result.append(s)
        # Skip blank lines.
        while i < len(lines) and not lines[i].strip():
            i += 1
        # Skip the pass line.
        if i < len(lines) and lines[i].lstrip().startswith('pass'):
            i += 1
        else:
            g.trace('*** no pass after %s: %s' % (kind, p.h))
    else:
        # A directive line.
        result.append(s)
    return i
#@+node:ekr.20180328065332.1: *3* Check conventions stuff
#@+node:ekr.20171208042251.1: *4* @@button check-conventions (no longer used)
g.cls()
if c.changed: c.save()

import imp
import leo.core.leoCheck as leoCheck
imp.reload(leoCheck)

do_all = True
do_string = True

fails = []
    # All of Leo's core files pass!
fn = g.os_path_finalize_join(g.app.loadDir, '..', 'core', 'leoTest.py')
<< define s >>
<< old tests >>
if do_all:
    utils = leoCheck.ProjectUtils()
    aList = utils.project_files('leo', force_all=False)
    # g.printList(aList)
    for fn in aList:
        sfn = g.shortFileName(fn)
        if sfn in fails:
            print('===== skipping', sfn)
        else:
            print('==== fn', sfn)
            leoCheck.ConventionChecker(c).check(fn=fn)
elif do_string: # Test string s.
    leoCheck.ConventionChecker(c).check(s=s)
else: # Test an actual file.
    leoCheck.ConventionChecker(c).check(fn=fn)
#@+node:ekr.20171208105236.1: *5* << define s >>
s = '''\
class T:
    
    def __init__(self, tempNode):
        self.tempNode = tempNode.copy()
    
    def setUp(self):
        tempNode = self.tempNode
        while tempNode.firstChild():
            tempNode.firstChild().doDelete()
'''

s_ok2 = '''
class Context(object):
    def __init__ (self, parent_context):
        self.parent_context = parent_context
        if parent_context:
            parent_context.inner_contexts_list.append(self)
'''

s_ok= '''
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
'''

#@+node:ekr.20171210062719.1: *5* << old tests >>
s_passes_1 = '''\
class C1:
        
    def f1(self, p):
        print(p.v)
        
    def f2(self, p):
        self.f1(p.v) # WRONG

'''


s_1 = '''\
class C1:

    def __init__(self, c):
        self.c = c
        c.theTagController = self
        
    def add_tag(self, p):
        pass

class C2:

    def oops(self, p):
        c.tagController.add_tag(p.v,tag)
            # WRONG: should be p.

'''


s_2 = '''\
class TagController:

    def __init__(self, c):
        self.c = c
        c.theTagController = self

    def add_tag(self, p, tag):
        # Will fail if p is a vnode
        tags = set(p.v.u.get('__node_tags', set([])))

class LeoTagWidget(QtWidgets.QWidget):

    def __init__(self,c,parent=None):
        self.c = c
        self.tc = self.c.theTagController

    def add_tag(self, event=None):
        p = self.c.p
        self.tc.add_tag(p.v,tag) # WRONG: should be p.

'''
#@+node:ekr.20160109150703.1: *4* class Stats (old & stupid, from )
class Stats(object):
    '''A class containing global statistics & other data'''
    @others
#@+node:ekr.20160109150703.2: *5*  sd.ctor
def __init__ (self):

    # Files...
    # self.completed_files = [] # Files handled by do_files.
    # self.failed_files = [] # Files that could not be opened.
    # self.files_list = [] # Files given by user or by import statements.
    # self.module_names = [] # Module names corresponding to file names.

    # Contexts.
    # self.context_list = {}
        # Keys are fully qualified context names; values are contexts.
    # self.modules_dict = {}
        # Keys are full file names; values are ModuleContext's.

    # Statistics...
    # self.n_chains = 0
    self.n_contexts = 0
    # self.n_errors = 0
    self.n_lambdas = 0
    self.n_modules = 0
    # self.n_relinked_pointers = 0
    # self.n_resolvable_names = 0
    # self.n_resolved_contexts = 0
    # self.n_relinked_names = 0

    # Names...
    self.n_attributes = 0
    self.n_expressions = 0
    self.n_ivars = 0
    self.n_names = 0        # Number of symbol table entries.
    self.n_del_names = 0
    self.n_load_names = 0
    self.n_param_names = 0
    self.n_param_refs = 0
    self.n_store_names = 0

    # Statements...
    self.n_assignments = 0
    self.n_calls = 0
    self.n_classes = 0
    self.n_defs = 0
    self.n_fors = 0
    self.n_globals = 0
    self.n_imports = 0
    self.n_lambdas = 0
    self.n_list_comps = 0
    self.n_returns = 0
    self.n_withs = 0

    # Times...
    self.parse_time = 0.0
    self.pass1_time = 0.0
    self.pass2_time = 0.0
    self.total_time = 0.0
#@+node:ekr.20160109150703.6: *5* sd.print_times
def print_times (self):

    sd = self
    times = (
        'parse_time',
        'pass1_time',
        # 'pass2_time', # the resolve_names pass is no longer used.
        'total_time',
    )
    max_n = 5
    for s in times:
        max_n = max(max_n,len(s))
    print('\nScan times...\n')
    for s in times:
        pad = ' ' * (max_n - len(s))
        print('%s%s: %2.2f' % (pad,s,getattr(sd,s)))
    print('')
#@+node:ekr.20160109150703.7: *5* sd.print_stats
def print_stats (self):

    sd = self
    table = (
        '*', 'errors',

        '*Contexts',
        'classes','contexts','defs','modules',

        '*Statements',
        'assignments','calls','fors','globals','imports',
        'lambdas','list_comps','returns','withs',

        '*Names',
        'attributes','del_names','load_names','names',
        'param_names','param_refs','store_names',
        #'resolvable_names','relinked_names','relinked_pointers',
        # 'ivars',
        # 'resolved_contexts',
    )
    max_n = 5
    for s in table:
        max_n = max(max_n,len(s))
    print('\nStatistics...\n')
    for s in table:
        var = 'n_%s' % s
        pad = ' ' * (max_n - len(s))
        if s.startswith('*'):
            if s[1:].strip():
                print('\n%s\n' % s[1:])
            else:
                pass # print('')
        else:
            pad = ' ' * (max_n - len(s))
            print('%s%s: %s' % (pad,s,getattr(sd,var)))
    print('')
#@+node:ekr.20171211054600.1: *4* OLD checkConventions (leoCheck.py)
def checkConventions(c):
    '''
    A stand-alone version of the @button node that tested the
    ConventionChecker class.
    
    The check-conventions command in checkerCommands.py saves c and reloads
    the leoCheck module before calling this function.
    '''
    g.cls()
    kind = 'all'
    project_name = 'leo'  # 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'
    assert kind in ('all', 'file', 'production', 'string'), repr(kind)
    fn = g.os_path_finalize_join(g.app.loadDir, '..', 'plugins', 'qt_tree.py')
    report_stats = True # and kind != 'production'
    trace_fn = True
    trace_skipped = False
    fails_dict = {
        'coverage': ['cmdline.py',],
        'lib2to3': ['fixer_util.py', 'fix_dict.py', 'patcomp.py', 'refactor.py'],
        'leo': [], # All of Leo's core files pass.
        'pylint': [
            'base.py', 'classes.py', 'format.py',
            'logging.py', 'python3.py', 'stdlib.py', 
            'docparams.py', 'lint.py',
        ],
        'rope': ['objectinfo.py', 'objectdb.py', 'runmod.py',],
    }
    fails = fails_dict.get(project_name, [])
    << define s >>
    s = g.adjustTripleString(s, c.tab_width)
    << old tests >>
    stats = Stats()
    if kind == 'production':
        for p in g.findRootsWithPredicate(c, c.p, predicate=None):
            x = ConventionChecker(c, stats)
            x.check(fn=g.fullPath(c, p), trace_fn=trace_fn)
    elif kind == 'all':
        utils = ProjectUtils()
        aList = utils.project_files(project_name, force_all=False)
        if aList:
            t1 = time.clock()
            for fn in aList:
                sfn = g.shortFileName(fn)
                if sfn in fails or fn in fails:
                    if trace_skipped: print('===== skipping', sfn)
                else:
                    ConventionChecker(c, stats).check(fn=fn, trace_fn=trace_fn)
            t2 = time.clock()
            print('%s files in %4.2f sec.' % (len(aList), (t2-t1)))
        else:
            print('no files for project: %s' % (project_name))
    elif kind == 'string':
        ConventionChecker(c, stats).check(s=s)
    else:
        assert kind == 'file', repr(kind)
        ConventionChecker(c, stats).check(fn=fn)
    if report_stats:
        stats.report()
#@+node:ekr.20171211054736.2: *5* << define s >>
s = '''\
class T:
    
    def __init__(self, tempNode):
        self.tempNode = tempNode.copy()
    
    def setUp(self):
        tempNode = self.tempNode
        while tempNode.firstChild():
            tempNode.firstChild().doDelete()
'''

s_ok2 = '''
class Context(object):
    def __init__ (self, parent_context):
        self.parent_context = parent_context
        if parent_context:
            parent_context.inner_contexts_list.append(self)
'''
assert s_ok2

s_ok= '''
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
'''
assert s_ok

#@+node:ekr.20171211054736.3: *5* << old tests >>
s_passes_1 = '''\
class C1:
        
    def f1(self, p):
        print(p.v)
        
    def f2(self, p):
        self.f1(p.v) # WRONG

'''
assert s_passes_1

s_1 = '''\
class C1:

    def __init__(self, c):
        self.c = c
        c.theTagController = self
        
    def add_tag(self, p):
        pass

class C2:

    def oops(self, p):
        c.tagController.add_tag(p.v,tag)
            # WRONG: should be p.

'''
assert s_1


s_2 = '''\
class TagController:

    def __init__(self, c):
        self.c = c
        c.theTagController = self

    def add_tag(self, p, tag):
        # Will fail if p is a vnode
        tags = set(p.v.u.get('__node_tags', set([])))

class LeoTagWidget(QtWidgets.QWidget):

    def __init__(self,c,parent=None):
        self.c = c
        self.tc = self.c.theTagController

    def add_tag(self, event=None):
        p = self.c.p
        self.tc.add_tag(p.v,tag) # WRONG: should be p.

'''
assert s_2
#@+node:ekr.20180813063846.1: *3* Fast-draw branches
#@+node:ekr.20180808075509.1: *4* qtree.partialDraw & helpers (never used)
def partialDraw(self, p):

    trace = True and not g.unitTesting
    c = self.c
    if 1:
        self.drawVisible()
    else:
        first_p = c.hoistStack[-1].p if c.hoistStack else c.rootPosition()
        aList1 = self.countVisible(first_p=first_p, target_p=p)
        aList2 = self.countVisible(first_p=p, target_p=None)
        if trace:
            n1, n2 = len(aList1), len(aList2)
            g.trace('%s + %s = %s' % (n1, n2, n1+n2))
        if 1:
            # Draw everything.
            self.drawList(aList1 + aList2)
        else:
            aList = self.computeVisiblePositions(aList1, aList2, p)
            self.drawList(aList)
#@+node:ekr.20180809110957.1: *5* qtree.computeVisiblePositions
def computeVisiblePositions(self, aList1, aList2, p):
    '''
    Compute the list of *visible* positions to be drawn.
    
    This is tricky.  We don't want to scroll the screen unnecessarily.
    '''
    # Show everything if possible.
    if len(aList1) + len(aList2) <= self.size:
        return aList1 + aList2
    c = self.c
    while p.hasParent():
        p.moveToParent()
    aList = []
    for i in range(self.size):
        aList.append(p.copy())
        p.moveToVisNext(c)
        if not p:
            break
    return aList
#@+node:ekr.20180809123937.1: *5* qtree.countVisible
def countVisible(self, first_p, target_p):
    """
    Return the number of visible positions from first_p to target_p.
    """
    c = self.c
    aList, p = [first_p.copy()], first_p.copy()
    while p:
        if p == target_p:
            return aList[:-1]
        v = p.v
        # if v.isExpanded() and v.hasChildren():
        if (v.statusBits & v.expandedBit) != 0 and v.children:
            # p.moveToFirstChild()
            p.stack.append((v, p._childIndex),)
            p.v = v.children[0]
            p._childIndex = 0
            aList.append(p.copy())
            continue
        # if p.hasNext():
        parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
        if p._childIndex + 1 < len(parent_v.children):
            # p.moveToNext()
            p._childIndex += 1
            p.v = parent_v.children[p._childIndex]
            aList.append(p.copy())
            continue
        #
        # A fast version of p.moveToThreadNext().
        # We look for a parent with a following sibling.
        while p.stack:
            # p.moveToParent()
            p.v, p._childIndex = p.stack.pop()
            # if p.hasNext():
            parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
            if p._childIndex + 1 < len(parent_v.children):
                # p.moveToNext()
                p._childIndex += 1
                p.v = parent_v.children[p._childIndex]
                break # Found: moveToThreadNext()
        else:
            break # Not found.
        # Found moveToThreadNext()
        aList.append(p.copy())
        continue
    if target_p:
        g.trace('NOT FOUND:', target_p.h)
    return aList
#@+node:ekr.20180809115019.1: *5* qtree.drawList
def drawList(self, aList):
    
    trace = False
    c = self.c
    parents = []
    # Clear the widget.
    w = self.treeWidget
    w.clear()
    # Clear the dicts.
    self.initData()
    for p in aList:
        level = p.level()
        parent_item = w if level == 0 else parents[level-1]
        item = QtWidgets.QTreeWidgetItem(parent_item)
        item.setFlags(item.flags() | QtCore.Qt.ItemIsEditable)
        item.setChildIndicatorPolicy(
            item.ShowIndicator if p.hasChildren()
            else item.DontShowIndicator)
        item.setExpanded(bool(p.hasChildren() and p.isExpanded()))
        self.items.append(item)
        if trace:
            print('')
            g.trace('===== level', level, p.h)
            g.trace('parent', id(parent_item), parent_item.__class__.__name__)
            g.trace('item  ', id(item), item.__class__.__name__)
            self.print_parents(parents, 1)
        # Update parents.
        if level == 0:
            parents = []
        else:
            parents = parents[:level]
        parents.append(item)
        if trace:
            self.print_parents(parents, 2)
        # Update the dicts.  Like rememberItem.
        itemHash = self.itemHash(item)
        self.item2positionDict[itemHash] = p.copy()
        self.item2vnodeDict[itemHash] = p.v
        self.position2itemDict[p.key()] = item
        d = self.vnode2itemsDict
        v = p.v
        aList = d.get(v, [])
        aList.append(item)
        d[v] = aList
        # Enter the headline.
        item.setText(0, p.h)
        # Set current item.
        if p == c.p:
            w.setCurrentItem(item)
#@+node:ekr.20180809111725.1: *5* qtree.printParents
def print_parents(self, parents, tag):
    print(tag)
    g.printObj([
        '%10s %s' % (id(z), z.__class__.__name__)
            for z in parents])
#@+node:ekr.20180810060655.1: *4* test vieldVisible
g.cls()
import time
tree = c.frame.tree
if 1: # works
    for p in c.all_positions():
        p.expand()
elif 1:
    for p in c.all_positions():
        p.v.expandedPositions = []
    for p in c.all_positions():
        p.v.expand()
        p.v.expandedPositions.append(p.copy())
else: # Doesn't work
    for v in c.all_nodes():
        v.expand()
    c.redraw() # This would be wrong.
if 1:
    t1 = time.clock()
    for i in range(1):
        aList1 = [z.copy() for z in tree.slowYieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('slow: %6.3f' % (t2-t1))
if 1:
    t1 = time.clock()
    for i in range(1):
        aList2 = [z.copy() for z in tree.yieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('fast: %6.3f' % (t2-t1))
v1 = [z.v for z in aList1]
v2 = [z.v for z in aList2]
try:
    i = 0
    while i < min(len(aList1), len(aList2)) and aList1[i] == aList2[i]:
        # print(i, aList1[i].h)
        i += 1
    if i == len(aList1) == len(aList2):
        print('OK')
    else:
        print('FAIL', i, len(aList1), len(aList2))
    assert aList1 == aList2, (len(aList1), len(aList2))
    assert v1 == v2, (len(v1), len(v2))
finally:
    for v in c.all_nodes():
        v.contract()
    c.redraw()
#@+node:ekr.20180810111515.1: *4* benchmark
import time
t1 = time.clock()
c.expandAllHeadlines()
t2 = time.clock()
print('expand: %5.2f sec' % (t2-t1))
t3 = time.clock()
tree = c.frame.tree
w = tree.treeWidget
n = 0
for p in tree.yieldVisible(c.rootPosition()):
    n += 1
    # c.selectPosition(p)
    item = tree.position2itemDict.get(c.p.key())
    if item:
        w.setCurrentItem(item)
t4 = time.clock()
print('%s nodes in %5.2f sec' % (n, t4-t3))
#@+node:ekr.20110605121601.17879: *4* qtree.rememberItem
def rememberItem(self, p, item):

    v = p.v
    # Update position dicts.
    itemHash = self.itemHash(item)
    self.position2itemDict[p.key()] = item
    self.item2positionDict[itemHash] = p.copy() # was item
    # Update item2vnodeDict.
    self.item2vnodeDict[itemHash] = v # was item
    # Update vnode2itemsDict.
    d = self.vnode2itemsDict
    aList = d.get(v, [])
    if item in aList:
        g.trace('*** ERROR *** item already in list: %s, %s' % (item, aList))
    else:
        aList.append(item)
    d[v] = aList
#@+node:ekr.20120219154958.10488: *4* LM.initFocusAndDraw (not used)
def initFocusAndDraw(self, c, fileName):

    def init_focus_handler(timer, c=c, p=c.p):
        '''Idle-time handler for initFocusAndDraw'''
        c.initialFocusHelper()
        c.outerUpdate()
        timer.stop()

    # This must happen after the code in getLeoFile.
    timer = g.IdleTime(init_focus_handler, delay=0.1, tag='getLeoFile')
    if timer:
        timer.start()
    else:
        # Default code.
        c.selectPosition(c.p)
        c.initialFocusHelper()
        c.k.showStateAndMode()
        c.outerUpdate()
#@+node:ekr.20150312225028.29: *3* leoViews project
This was a major project, now abandoned.
#@+node:ekr.20150312225028.31: *4* class OrganizerData
class OrganizerData:
    '''A class containing all data for a particular organizer node.'''
    def __init__ (self,h,unl,unls):
        self.anchor = None # The anchor position of this od node.
        self.children = [] # The direct child od nodes of this od node.
        self.closed = False # True: this od node no longer accepts new child od nodes.
        self.drop = True # Drop the unl for this od node when associating positions with unls.
        self.descendants = None # The descendant od nodes of this od node.
        self.exists = False # True: this od was created by @existing-organizer:
        self.h = h # The headline of this od node.
        self.moved = False # True: the od node has been moved to a global move list.
        self.opened = False # True: the od node has been opened.
        self.organized_nodes = [] # The list of positions organized by this od node.
        self.parent_od = None # The parent od node of this od node. (None is valid.)
        self.p = None # The position of this od node.
        self.parent = None # The original parent position of all nodes organized by this od node.
            # If parent_od is None, this will be the parent position of the od node.
        self.source_unl = None # The unl of self.parent.
        self.unl = unl # The unl of this od node.
        self.unls = unls # The unls contained in this od node.
        self.visited = False # True: demote_helper has already handled this od node.
    def __repr__(self):
        return 'OrganizerData: %s' % (self.h or '<no headline>')
    __str__ = __repr__
#@+node:ekr.20150312225028.32: *4* class ViewController
class ViewController:
    << docstring >>
    @others
#@+node:ekr.20150312225028.33: *5*  << docstring >> (class ViewController)
'''
A class to handle @views trees and related operations.
Such trees have the following structure:

- @views
  - @auto-view <unl of @auto node>
    - @organizers
      - @organizer <headline>
    - @clones
    
The body text of @organizer and @clones consists of unl's, one per line.
'''
#@+node:ekr.20150312225028.34: *5*  vc.ctor & vc.init
def __init__ (self,c):
    '''Ctor for ViewController class.'''
    self.c = c
    self.headline_ivar = '_imported_headline'
    self.init()
    
def init(self):
    '''
    Init all ivars of this class.
    Unit tests may call this method to ensure that this class is re-inited properly.
    '''
    self.all_ods = []
        # List of all od nodes.
    self.anchors_d = {}
        # Keys are anchoring positions, values are sorted lists of ods.
    self.anchor_offset_d = {}
        # Keys are anchoring positions, values are ints.
    self.existing_ods = []
        # List of od instances corresponding to @existing-organizer: nodes.
    self.global_bare_organizer_node_list = []
        # List of organizers that have no parent organizer node.
        # This list excludes existing organizer nodes.
    self.headlines_dict = {}
        # Keys are vnodes; values are list of child headlines.
    self.imported_organizers_list = []
        # The list of nodes that have children on entry, such as class nodes.
    self.n_nodes_scanned = 0
        # Number of nodes scanned by demote.
    self.organizer_ods = []
        # List of od instances corresponding to @organizer: nodes.
    self.organizer_unls = []
        # The list of od.unl for all od instances in self.organizer_ods.
    self.root = None
        # The position of the @auto node.
    self.pending = []
        # The list of nodes pending to be added to an organizer.
    self.stack = []
        # The stack containing real and virtual parent nodes during the main loop.
    self.temp_node = None
        # The parent position of all holding cells.
    self.trail_write_1 = None
        # The trial write on entry.
    self.views_node = None
        # The position of the @views node.
    self.work_list = []
        # A gloal list of (parent,child) tuples for all nodes that are
        # to be moved to **non-existing** organizer nodes.
        # **Important**: Nodes are moved in the order they appear in this list:
        # the tuples contain no childIndex component!
        # This list is the "backbone" of this class:
        # - The front end (demote and its helpers) adds items to this list.
        # - The back end (move_nodes and its helpers) moves nodes using this list.
#@+node:ekr.20150312225028.35: *5* vc.Entry points
#@+node:ekr.20150312225028.36: *6* vc.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self,root):
    # Define class ConvertController.
    @others
    vc = self
    c = vc.c
    if root.isAtFileNode():
        ConvertController(c,root).run()
    else:
        g.es_print('not an @file node:',root.h)
#@+node:ekr.20150312225028.37: *7* class ConvertController
class ConvertController:
    def __init__ (self,c,p):
        self.c = c
        # self.ic = c.importCommands
        self.vc = c.viewController
        self.root = p.copy()
    @others
#@+node:ekr.20150312225028.38: *8* cc.delete_at_auto_view_nodes
def delete_at_auto_view_nodes(self,root):
    '''Delete all @auto-view nodes pertaining to root.'''
    cc = self
    vc = cc.vc
    while True:
        p = vc.has_at_auto_view_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20150312225028.39: *8* cc.import_from_string
def import_from_string(self,s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    scanner = ic.scanner_for_ext(ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True,parent=p,s=s)
    p.h = root.h.replace('@file','@auto' if ok else '@@auto')
    return ok,p
#@+node:ekr.20150312225028.40: *8* cc.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root,vc = cc.root,c.viewController
    # set the headline_ivar for all vnodes.
    t1 = time.clock()
    cc.set_expected_imported_headlines(root)
    t2 = time.clock()
    # Delete all previous @auto-view nodes for this tree.
    cc.delete_at_auto_view_nodes(root)
    t3 = time.clock()
    # Ensure that all nodes of the tree are regularized.
    ok = vc.prepass(root)
    t4 = time.clock()
    if not ok:
        g.es_print('Can not convert',root.h,color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
            # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4-t3),
            '\n  total:                           %4.2f sec' % (t4-t1))
        return
    # Create the appropriate @auto-view node.
    at_auto_view = vc.update_before_write_at_auto_file(root)
    t5 = time.clock()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.clock()
    if trace and trace_s:
        g.trace('source file...\n',s)
    # Import the @auto string.
    ok,p = cc.import_from_string(s)
    t7 = time.clock()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = vc.at_auto_view_body(p)
        # Recreate the organizer nodes, headlines, etc.
        ok = vc.update_after_read_at_auto_file(p)
        t8 = time.clock()
        if not ok:
            p.h = '@@' + p.h
            g.trace('restoring original @auto file')
            ok,p = cc.import_from_string(s)
            if ok:
                p.h = '@@' + p.h + ' (restored)'
                if p.next():
                    p.moveAfter(p.next())
        t9 = time.clock()
    else:
        t8 = t9 = time.clock()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
        # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4-t3),
        '\n  update_before_write_at_auto_file:%4.2f sec' % (t5-t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6-t5),
        '\n  import_from_string:              %4.2f sec' % (t7-t6),
        '\n  update_after_read_at_auto_file   %4.2f sec' % (t8-t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9-t8),
        '\n  total:                           %4.2f sec' % (t9-t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20150312225028.41: *8* cc.set_expected_imported_headlines
def set_expected_imported_headlines(self,root):
    '''Set the headline_ivar for all vnodes.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    aClass = ic.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic,atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c,root,importing=True)
    fn = c.os_path_finalize_join(theDir,root.h)
    fn = root.h.replace('\\','/')
    junk,fn = g.os_path_split(fn)
    fn,junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner,'headlineForNode'):
        ivar = cc.vc.headline_ivar
        for p in root.subtree():
            if not hasattr(p.v,ivar):
                h = scanner.headlineForNode(fn,p)
                setattr(p.v,ivar,h)
                if trace and h != p.h:
                    g.trace('==>',h) # p.h,'==>',h
#@+node:ekr.20150312225028.42: *8* cc.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind = '@file',
        nosentinels = True,
        perfectImportFlag = False,
        scriptWrite = False,
        thinFile = True,
        toString = True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:',ok,'s:...\n'+s)
    return s
#@+node:ekr.20150312225028.43: *6* vc.pack & helper
def pack(self):
    '''
    Undoably convert c.p to a packed @view node, replacing all cloned
    children of c.p by unl lines in c.p.b.
    '''
    vc = self
    c,u = vc.c,vc.c.undoer
    vc.init()
    changed = False
    root = c.p
    # Create an undo group to handle changes to root and @views nodes.
    # Important: creating the @views node does *not* invalidate any positions.'''
    u.beforeChangeGroup(root,'view-pack')
    if not vc.has_at_views_node():
        changed = True
        bunch = u.beforeInsertNode(c.rootPosition())
        views = vc.find_at_views_node()
            # Creates the @views node as the *last* top-level node
            # so that no positions become invalid as a result.
        u.afterInsertNode(views,'create-views-node',bunch)
    # Prepend @view if need.
    if not root.h.strip().startswith('@'):
        changed = True
        bunch = u.beforeChangeNodeContents(root)
        root.h = '@view ' + root.h.strip()
        u.afterChangeNodeContents(root,'view-pack-update-headline',bunch)
    # Create an @view node as a clone of the @views node.
    bunch = u.beforeInsertNode(c.rootPosition())
    new_clone = vc.create_view_node(root)
    if new_clone:
        changed = True
        u.afterInsertNode(new_clone,'create-view-node',bunch)
    # Create a list of clones that have a representative node
    # outside of the root's tree.
    reps = [vc.find_representative_node(root,p)
        for p in root.children()
            if vc.is_cloned_outside_parent_tree(p)]
    reps = [z for z in reps if z is not None]
    if reps:
        changed = True
        bunch = u.beforeChangeTree(root)
        c.setChanged(True)
        # Prepend a unl: line for each cloned child.
        unls = ['unl: %s\n' % (vc.unl(p)) for p in reps]
        root.b = ''.join(unls) + root.b
        # Delete all child clones in the reps list.
        v_reps = set([p.v for p in reps])
        while True:
            for child in root.children():
                if child.v in v_reps:
                    child.doDelete()
                    break
            else: break
        u.afterChangeTree(root,'view-pack-tree',bunch)
    if changed:
        u.afterChangeGroup(root,'view-pack')
        c.selectPosition(root)
        c.redraw()
#@+node:ekr.20150312225028.44: *7* vc.create_view_node
def create_view_node(self,root):
    '''
    Create a clone of root as a child of the @views node.
    Return the *newly* cloned node, or None if it already exists.
    '''
    vc = self
    c = vc.c
    # Create a cloned child of the @views node if it doesn't exist.
    views = vc.find_at_views_node()
    for p in views.children():
        if p.v == c.p.v:
            return None
    p = root.clone()
    p.moveToLastChildOf(views)
    return p
#@+node:ekr.20150312225028.45: *6* vc.unpack
def unpack(self):
    '''
    Undoably unpack nodes corresponding to leading unl lines in c.p to child clones.
    Return True if the outline has, in fact, been changed.
    '''
    vc = self
    c,root,u = vc.c,vc.c.p,vc.c.undoer
    vc.init()
    # Find the leading unl: lines.
    i,lines,tag = 0,g.splitLines(root.b),'unl:'
    for s in lines:
        if s.startswith(tag): i += 1
        else: break
    changed = i > 0
    if changed:
        bunch = u.beforeChangeTree(root)
        # Restore the body
        root.b = ''.join(lines[i:])
        # Create clones for each unique unl.
        unls = list(set([s[len(tag):].strip() for s in lines[:i]]))
        for unl in unls:
            p = vc.find_absolute_unl_node(unl)
            if p: p.clone().moveToLastChildOf(root)
            else: g.trace('not found: %s' % (unl))
        c.setChanged(True)
        c.undoer.afterChangeTree(root,'view-unpack',bunch)
        c.redraw()
    return changed
#@+node:ekr.20150312225028.46: *6* vc.update_before_write_at_auto_file
def update_before_write_at_auto_file(self,root):
    '''
    Update the @auto-view node for root, an @auto node. Create @organizer,
    @existing-organizer, @clones and @headlines nodes as needed.
    This *must not* be called for trial writes.
    '''
    trace = False and not g.unitTesting
    vc = self
    c = vc.c
    changed = False
    t1 = time.clock()
    # Create lists of cloned and organizer nodes.
    clones,existing_organizers,organizers = \
        vc.find_special_nodes(root)
    # Delete all children of the @auto-view node for this @auto node.
    at_auto_view = vc.find_at_auto_view_node(root)
    if at_auto_view.hasChildren():
        changed = True
        at_auto_view.deleteAllChildren()
    # Create the single @clones node.
    if clones:
        at_clones = vc.find_at_clones_node(root)
        at_clones.b = ''.join(
            ['gnx: %s\nunl: %s\n' % (z[0],z[1]) for z in clones])
    # Create the single @organizers node.
    if organizers or existing_organizers:
        at_organizers = vc.find_at_organizers_node(root)
    # Create one @organizers: node for each organizer node.
    for p in organizers:
        # g.trace('organizer',p.h)
        at_organizer = at_organizers.insertAsLastChild()
        at_organizer.h = '@organizer: %s' % p.h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create one @existing-organizer node for each existing organizer.
    ivar = vc.headline_ivar
    for p in existing_organizers:
        at_organizer = at_organizers.insertAsLastChild()
        h = getattr(p.v,ivar,p.h)
        if trace and h != p.h: g.trace('==>',h) # p.h,'==>',h
        at_organizer.h = '@existing-organizer: %s' % h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create the single @headlines node.
    vc.create_at_headlines(root)
    if changed and not g.unitTesting:
        g.es_print('updated @views node in %4.2f sec.' % (
            time.clock()-t1))
    if changed:
        c.redraw()
    return at_auto_view # For at-file-to-at-auto command.
#@+node:ekr.20150312225028.47: *7* vc.create_at_headlines
def create_at_headlines(self,root):
    '''Create the @headlines node for root, an @auto file.'''
    vc = self
    c = vc.c
    result = []
    ivar = vc.headline_ivar
    for p in root.subtree():
        h = getattr(p.v,ivar,None)
        if h is not None and p.h != h:
            # g.trace('custom:',p.h,'imported:',h)
            unl = vc.relative_unl(p,root)
            aList = unl.split('-->')
            aList[-1] = h
            unl = '-->'.join(aList)
            result.append('imported unl: %s\nhead: %s\n' % (
                unl,p.h))
            delattr(p.v,ivar)
    if result:
        p = vc.find_at_headlines_node(root)
        p.b = ''.join(result)
#@+node:ekr.20150312225028.48: *7* vc.find_special_nodes
def find_special_nodes(self,root):
    '''
    Scan root's tree, looking for organizer and cloned nodes.
    Exclude organizers on imported organizers list.
    '''
    trace = False and not g.unitTesting
    verbose = False
    vc = self
    clones,existing_organizers,organizers = [],[],[]
    if trace: g.trace('imported existing',
        [v.h for v in vc.imported_organizers_list])
    for p in root.subtree():
        if p.isCloned():
            rep = vc.find_representative_node(root,p)
            if rep:
                unl = vc.relative_unl(p,root)
                gnx = rep.v.gnx
                clones.append((gnx,unl),)
        if p.v in vc.imported_organizers_list:
            # The node had children created by the importer.
            if trace and verbose: g.trace('ignore imported existing',p.h)
        elif vc.is_organizer_node(p,root):
            # p.hasChildren and p.b is empty, except for comments.
            if trace and verbose: g.trace('organizer',p.h)
            organizers.append(p.copy())
        elif p.hasChildren():
            if trace and verbose: g.trace('existing',p.h)
            existing_organizers.append(p.copy())
    return clones,existing_organizers,organizers
#@+node:ekr.20150312225028.49: *6* vc.update_after_read_at_auto_file & helpers
def update_after_read_at_auto_file(self,root):
    '''
    Recreate all organizer nodes and clones for a single @auto node
    using the corresponding @organizer: and @clones nodes.
    '''
    trace = True and not g.unitTesting
    vc = self
    c = vc.c
    if not vc.is_at_auto_node(root):
        return # Not an error: it might be and @auto-rst node.
    old_changed = c.isChanged()
    try:
        vc.init()
        vc.root = root.copy()
        t1 = time.clock()
        vc.trial_write_1 = vc.trial_write(root)
        t2 = time.clock()
        at_organizers = vc.has_at_organizers_node(root)
        t3 = time.clock()
        if at_organizers:
            vc.create_organizer_nodes(at_organizers,root)
        t4 = time.clock()
        at_clones = vc.has_at_clones_node(root)
        if at_clones:
            vc.create_clone_links(at_clones,root)
        t5 = time.clock()
        n = len(vc.work_list)
        ok = vc.check(root)
        t6 = time.clock()
        if ok:
            vc.update_headlines_after_read(root)
        t7 = time.clock()
        c.setChanged(old_changed if ok else False)
            # To do: revert if not ok.
    except Exception:
        g.es_exception()
        n = 0
        ok = False
    if trace:
        if t7-t1 > 0.5: g.trace(
            '\n  trial_write:                 %4.2f sec' % (t2-t1),
            # '\n  has_at_organizers_node:    %4.2f sec' % (t3-t2),
            '\n  create_organizer_nodes:      %4.2f sec' % (t4-t3),
            '\n  create_clone_links:          %4.2f sec' % (t5-t4),
            '\n  check:                       %4.2f sec' % (t6-t5),
            '\n  update_headlines_after_read: %4.2f sec' % (t7-t6),
            '\n  total:                       %4.2f sec' % (t7-t1))
            # '\n  file:',root.h)
        # else: g.trace('total: %4.2f sec' % (t7-t1),root.h)
    if ok and n > 0:
        g.es('rearragned: %s' % (root.h),color='blue')
        g.es('moved %s nodes in %4.2f sec.' % (n,t7-t1))
        g.trace('@auto-view moved %s nodes in %4.2f sec. for' % (
            n,t2),root.h,noname=True)
    c.selectPosition(root)
    c.redraw()
    return ok
#@+node:ekr.20150312225028.50: *7* vc.check
def check (self,root):
    '''
    Compare a trial write or root with the vc.trail_write_1.
    Unlike the perfect-import checks done by the importer,
    we expecct an *exact* match, regardless of language.
    '''
    trace = True # and not g.unitTesting
    vc = self
    trial1 = vc.trial_write_1
    trial2 = vc.trial_write(root)
    if trial1 != trial2:
        g.pr('') # Don't use print: it does not appear with the traces.
        g.es_print('perfect import check failed for:',color='red')
        g.es_print(root.h,color='red')
        if trace:
            vc.compare_trial_writes(trial1,trial2)
            g.pr('')
    return trial1 == trial2
#@+node:ekr.20150312225028.51: *7* vc.create_clone_link
def create_clone_link(self,gnx,root,unl):
    '''
    Replace the node in the @auto tree with the given unl by a
    clone of the node outside the @auto tree with the given gnx.
    '''
    trace = False and not g.unitTesting
    vc = self
    p1 = vc.find_position_for_relative_unl(root,unl)
    p2 = vc.find_gnx_node(gnx)
    if p1 and p2:
        if trace: g.trace('relink',gnx,p2.h,'->',p1.h)
        if p1.b == p2.b:
            p2._relinkAsCloneOf(p1)
            return True
        else:
            g.es('body text mismatch in relinked node',p1.h)
            return False
    else:
        if trace: g.trace('relink failed',gnx,root.h,unl)
        return False
#@+node:ekr.20150312225028.52: *7* vc.create_clone_links
def create_clone_links(self,at_clones,root):
    '''
    Recreate clone links from an @clones node.
    @clones nodes contain pairs of lines (gnx,unl)
    '''
    vc = self
    lines = g.splitLines(at_clones.b)
    gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')]
    unls = [s[4:].strip() for s in lines if s.startswith('unl:')]
    # g.trace('at_clones.b',at_clones.b)
    if len(gnxs) == len(unls):
        vc.headlines_dict = {} # May be out of date.
        ok = True
        for gnx,unl in zip(gnxs,unls):
            ok = ok and vc.create_clone_link(gnx,root,unl)
        return ok
    else:
        g.trace('bad @clones contents',gnxs,unls)
        return False
#@+node:ekr.20150312225028.53: *7* vc.create_organizer_nodes & helpers
def create_organizer_nodes(self,at_organizers,root):
    '''
    root is an @auto node. Create an organizer node in root's tree for each
    child @organizer: node of the given @organizers node.
    '''
    vc = self
    c = vc.c
    trace = False and not g.unitTesting
    t1 = time.clock()
    vc.pre_move_comments(root)
        # Merge comment nodes with the next node.
    t2 = time.clock()
    vc.precompute_all_data(at_organizers,root)
        # Init all data required for reading.
    t3 = time.clock()
    vc.demote(root)
        # Traverse root's tree, adding nodes to vc.work_list.
    t4 = time.clock()
    vc.move_nodes()
        # Move nodes on vc.work_list to their final locations.
    t5 = time.clock()
    vc.post_move_comments(root)
        # Move merged comments to parent organizer nodes.
    t6 = time.clock()
    if trace: g.trace(
        '\n  pre_move_comments:   %4.2f sec' % (t2-t1),
        '\n  precompute_all_data: %4.2f sec' % (t3-t2),
        '\n  demote:              %4.2f sec' % (t4-t3),
        '\n  move_nodes:          %4.2f sec' % (t5-t4),
        '\n  post_move_comments:  %4.2f sec' % (t6-t5))
#@+node:ekr.20150312225028.54: *7* vc.update_headlines_after_read
def update_headlines_after_read(self,root):
    '''Handle custom headlines for all imported nodes.'''
    trace = False and not g.unitTesting
    vc = self
    # Remember the original imported headlines.
    ivar = vc.headline_ivar
    for p in root.subtree():
        if not hasattr(p.v,ivar):
            setattr(p.v,ivar,p.h)
    # Update headlines from @headlines nodes.
    at_headlines = vc.has_at_headlines_node(root)
    tag1,tag2 = 'imported unl: ','head: '
    n1,n2 = len(tag1),len(tag2)
    if at_headlines:
        lines = g.splitLines(at_headlines.b)
        unls  = [s[n1:].strip() for s in lines if s.startswith(tag1)]
        heads = [s[n2:].strip() for s in lines if s.startswith(tag2)]
    else:
        unls,heads = [],[]
    if len(unls) == len(heads):
        vc.headlines_dict = {} # May be out of date.
        for unl,head in zip(unls,heads):
            p = vc.find_position_for_relative_unl(root,unl)
            if p:
                if trace: g.trace('unl:',unl,p.h,'==>',head)
                p.h = head
    else:
        g.trace('bad @headlines body',at_headlines.b)
#@+node:ekr.20150312225028.55: *5* vc.Main Lines
#@+node:ekr.20150312225028.56: *6* vc.precompute_all_data & helpers
def precompute_all_data(self,at_organizers,root):
    '''Precompute all data needed to reorganize nodes.'''
    trace = False and not g.unitTesting
    vc = self
    t1 = time.clock() 
    vc.find_imported_organizer_nodes(root)
        # Put all nodes with children on vc.imported_organizer_node_list
    t2 = time.clock()
    vc.create_organizer_data(at_organizers,root)
        # Create OrganizerData objects for all @organizer:
        # and @existing-organizer: nodes.
    t3 = time.clock()
    vc.create_actual_organizer_nodes()
        # Create the organizer nodes in holding cells so positions remain valid.
    t4 = time.clock()
    vc.create_tree_structure(root)
        # Set od.parent_od, od.children & od.descendants for all ods.
    t5 = time.clock()
    vc.compute_all_organized_positions(root)
        # Compute the positions organized by each organizer.
        # ** Most of the time is spent here **.
    t6 = time.clock()
    vc.create_anchors_d()
        # Create the dictionary that associates positions with ods.
    t7 = time.clock()
    if trace: g.trace(
        '\n  find_imported_organizer_nodes:   %4.2f sec' % (t2-t1),
        '\n  create_organizer_data:           %4.2f sec' % (t3-t2),
        '\n  create_actual_organizer_nodes:   %4.2f sec' % (t4-t3),
        '\n  create_tree_structure:           %4.2f sec' % (t5-t4),
        '\n  compute_all_organized_positions: %4.2f sec' % (t6-t5),
        '\n  create_anchors_d:                %4.2f sec' % (t7-t6))
#@+node:ekr.20150312225028.57: *7* 1: vc.find_imported_organizer_nodes
def find_imported_organizer_nodes(self,root):
    '''
    Put the VNode of all imported nodes with children on
    vc.imported_organizers_list.
    '''
    trace = False # and not g.unitTesting
    vc = self
    aList = []
    for p in root.subtree():
        if p.hasChildren():
            aList.append(p.v)
    vc.imported_organizers_list = list(set(aList))
    if trace: g.trace([z.h for z in vc.imported_organizers_list])
#@+node:ekr.20150312225028.58: *7* 2: vc.create_organizer_data (od.p & od.parent)
def create_organizer_data(self,at_organizers,root):
    '''
    Create OrganizerData nodes for all @organizer: and @existing-organizer:
    nodes in the given @organizers node.
    '''
    vc = self
    vc.create_ods(at_organizers)
    vc.finish_create_organizers(root)
    vc.finish_create_existing_organizers(root)
    for od in vc.all_ods:
        assert od.parent,(od.exists,od.h)
#@+node:ekr.20150312225028.59: *8* vc.create_ods
def create_ods(self,at_organizers):
    '''Create all organizer nodes and the associated lists.'''
    # Important: we must completely reinit all data here.
    vc = self
    tag1 = '@organizer:'
    tag2 = '@existing-organizer:'
    vc.all_ods,vc.existing_ods,vc.organizer_ods = [],[],[]
    for at_organizer in at_organizers.children():
        h = at_organizer.h
        for tag in (tag1,tag2):
            if h.startswith(tag):
                unls = vc.get_at_organizer_unls(at_organizer)
                if unls:
                    organizer_unl = vc.drop_unl_tail(unls[0])
                    h = h[len(tag):].strip()
                    od = OrganizerData(h,organizer_unl,unls)
                    vc.all_ods.append(od)
                    if tag == tag1:
                        vc.organizer_ods.append(od)
                        vc.organizer_unls.append(organizer_unl)
                    else:
                        vc.existing_ods.append(od)
                        # Do *not* append organizer_unl to the unl list.
                else:
                    g.trace('===== no unls:',at_organizer.h)
#@+node:ekr.20150312225028.60: *8* vc.finish_create_organizers
def finish_create_organizers(self,root):
    '''Finish creating all organizers.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.organizer_ods[:]: 
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.parent = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.parent:
            od.anchor = od.parent
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This is, most likely, a true error.
            g.trace('===== removing od:',od.h)
            vc.organizer_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods
#@+node:ekr.20150312225028.61: *8* vc.finish_create_existing_organizers
def finish_create_existing_organizers(self,root):
    '''Finish creating existing organizer nodes.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.existing_ods[:]:
        od.exists = True
        assert od.unl not in vc.organizer_unls
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.p = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.p:
            od.anchor = od.p
            assert od.p.h == od.h,(od.p.h,od.h)  
            od.parent = od.p # Here, od.parent represents the "source" p.
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This arises when the imported node name doesn't match.
            g.trace('===== removing existing organizer:',od.h)
            vc.existing_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods

#@+node:ekr.20150312225028.62: *7* 3: vc.create_actual_organizer_nodes
def create_actual_organizer_nodes(self):
    '''
    Create all organizer nodes as children of holding cells. These holding
    cells ensure that moving an organizer node leaves all other positions
    unchanged.
    '''
    vc = self
    c = vc.c
    last = c.lastTopLevel()
    temp = vc.temp_node = last.insertAfter()
    temp.h = 'ViewController.temp_node'
    for od in vc.organizer_ods:
        holding_cell = temp.insertAsLastChild()
        holding_cell.h = 'holding cell for ' + od.h
        od.p = holding_cell.insertAsLastChild()
        od.p.h = od.h
#@+node:ekr.20150312225028.63: *7* 4: vc.create_tree_structure & helper
def create_tree_structure(self,root):
    '''Set od.parent_od, od.children & od.descendants for all ods.'''
    trace = False and not g.unitTesting
    vc = self
    # if trace: g.trace([z.h for z in data_list],g.callers())
    organizer_unls = [z.unl for z in vc.all_ods]
    for od in vc.all_ods:
        for unl in od.unls:
            if unl in organizer_unls:
                i = organizer_unls.index(unl)
                d2 = vc.all_ods[i]
                # if trace: g.trace('found organizer unl:',od.h,'==>',d2.h)
                od.children.append(d2)
                d2.parent_od = od
    # create_organizer_data now ensures od.parent is set.
    for od in vc.all_ods:
        assert od.parent,od.h
    # Extend the descendant lists.
    for od in vc.all_ods:
        vc.compute_descendants(od)
        assert od.descendants is not None
    if trace:
        def tail(head,unl):
            return str(unl[len(head):]) if unl.startswith(head) else str(unl)
        for od in vc.all_ods:
            g.trace(
                '\n  od:',od.h,
                '\n  unl:',od.unl,
                '\n  unls:', [tail(od.unl,z) for z in od.unls],
                '\n  source (unl):',od.source_unl or repr(''),
                '\n  parent (pos):', od.parent.h,
                '\n  children:',[z.h for z in od.children],
                '\n  descendants:',[str(z.h) for z in od.descendants])
#@+node:ekr.20150312225028.64: *8* vc.compute_descendants
def compute_descendants(self,od,level=0,result=None):
    '''Compute the descendant od nodes of od.'''
    trace = False # and not g.unitTesting
    vc = self
    if level == 0:
        result = []
    if od.descendants is None:
        for child in od.children:
            result.append(child)
            result.extend(vc.compute_descendants(child,level+1,result))
            result = list(set(result))
        if level == 0:
            od.descendants = result
            if trace: g.trace(od.h,[z.h for z in result])
        return result
    else:
        if trace: g.trace('cached',od.h,[z.h for z in od.descendants])
        return od.descendants
#@+node:ekr.20150312225028.65: *7* 5: vc.compute_all_organized_positions
def compute_all_organized_positions(self,root):
    '''Compute the list of positions organized by every od.'''
    trace = False and not g.unitTesting
    vc = self
    for od in vc.all_ods:
        if od.unls:
            # Do a full search only for the first unl.
            # parent = vc.find_position_for_relative_unl(root,od.unls[0])
            if True: # parent:
                for unl in od.unls:
                    p = vc.find_position_for_relative_unl(root,unl)
                    # p = vc.find_position_for_relative_unl(parent,vc.unl_tail(unl))
                    if p:
                        od.organized_nodes.append(p.copy())
                    if trace: g.trace('exists:',od.exists,
                        'od:',od.h,'unl:',unl,
                        'p:',p and p.h or '===== None')
            else:
                g.trace('fail',od.unls[0])
#@+node:ekr.20150312225028.66: *7* 6: vc.create_anchors_d
def create_anchors_d (self):
    '''
    Create vc.anchors_d.
    Keys are positions, values are lists of ods having that anchor.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = {}
    if trace: g.trace('all_ods',[z.h for z in vc.all_ods])
    for od in vc.all_ods:
        # Compute the anchor if it does not yet exists.
        # Valid now that p.__hash__ exists.
        key = od.anchor
        # key = '.'.join([str(z) for z in od.anchor.sort_key(od.anchor)])
        # key = '%s (%s)' % (key,od.anchor.h)
        aList = d.get(key,[])
        # g.trace(od.h,od.anchor.h,key,aList)
        aList.append(od)
        d[key] = aList
    if trace:
        for key in sorted(d.keys()):
            g.trace('od.anchor: %s ods: [%s]' % (key.h,','.join(z.h for z in d.get(key))))
    vc.anchors_d = d
#@+node:ekr.20150312225028.67: *6* vc.demote & helpers
def demote(self,root):
    '''
    The main line of the @auto-view algorithm. Traverse root's entire tree,
    placing items on the global work list.
    '''
    trace = False # and not g.unitTesting
    trace_loop = True
    vc = self
    active = None # The active od.
    vc.pending = [] # Lists of pending demotions.
    d = vc.anchor_offset_d # For traces.
    for p in root.subtree():
        parent = p.parent()
        if trace and trace_loop:
            if 1:
                g.trace('-----',p.childIndex(),p.h)
            else:
                g.trace(
                    '=====\np:',p.h,
                    'childIndex',p.childIndex(),
                    '\nparent:',parent.h,
                    'parent:offset',d.get(parent,0))
        vc.n_nodes_scanned += 1
        vc.terminate_organizers(active,parent)
        found = vc.find_organizer(parent,p)
        if found:
            pass # vc.enter_organizers(found,p)
        else:
            pass # vc.terminate_all_open_organizers()
        if trace and trace_loop:
            g.trace(
                'active:',active and active.h or 'None',
                'found:',found and found.h or 'None')
        # The main case statement...
        if found is None and active:
            vc.add_to_pending(active,p)
        elif found is None and not active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(None,p)
        elif found and found == active:
            # Pending nodes *will* be organized.
            for z in vc.pending:
                active2,child2 = z
                vc.add(active2,child2,'found==active:pending')
            vc.pending = []
            vc.add(active,p,'found==active')
        elif found and found != active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(found,p)
            active = found
            vc.enter_organizers(found,p)
            vc.add(active,p,'found!=active')
        else: assert False,'can not happen'
#@+node:ekr.20150312225028.68: *7* vc.add
def add(self,active,p,tag):
    '''
    Add p, an existing (imported) node to the global work list.
    Subtract 1 from the vc.anchor_offset_d entry for p.parent().
    
    Exception: do *nothing* if p is a child of an existing organizer node.
    '''
    trace = False # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(active,g.callers())
    if active.p == p.parent() and active.exists:
        if trace and verbose: g.trace('===== do nothing',active.h,p.h)
    else:
        data = active.p,p.copy()
        vc.add_to_work_list(data,tag)
        vc.anchor_decr(anchor=p.parent(),p=p)
        
#@+node:ekr.20150312225028.69: *7* vc.add_organizer_node
def add_organizer_node (self,od,p):
    '''
    Add od to the appropriate move list.
    p is the existing node that caused od to be added.
    '''
    trace = True # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(od.h,'parent',od.parent_od and od.parent_od.h or 'None')
    if od.parent_od:
        # Not a bare organizer: a child of another organizer node.
        # If this is an existing organizer, it's *position* may have
        # been moved without active.moved being set.
        data = od.parent_od.p,od.p
        if data in vc.work_list:
            if trace and verbose: g.trace(
                '**** duplicate 1: setting moved bit.',od.h)
            od.moved = True
        elif od.parent_od.exists:    
            anchor = od.parent_od.p
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            # g.trace('anchor:',anchor.h,'p:',p.h,'childIndex',p.childIndex())
            vc.add_to_bare_list(data,'non-bare existing')
        else:
            vc.add_to_work_list(data,'non-bare')
    elif od.p == od.anchor:
        if trace and verbose: g.trace(
            '***** existing organizer: do not move:',od.h)
    else:
        # This can be pre-computed?
        bare_list = [p for parent,p,n in vc.global_bare_organizer_node_list]
        if od.p in bare_list:
            if trace and verbose: g.trace(
                '**** duplicate 2: setting moved bit.',od.h)
            od.moved = True
        else:
            # A bare organizer node: a child of an *ordinary* node.
            anchor = p.parent()
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            vc.add_to_bare_list(data,'bare')
#@+node:ekr.20150312225028.70: *7* vc.add_to_bare_list
def add_to_bare_list(self,data,tag):
    '''Add data to the bare organizer list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    # Prevent duplicagtes.
    anchor,p,n = data
    for data2 in vc.global_bare_organizer_node_list:
        a2,p2,n2 = data2
        if p == p2:
            if trace: g.trace('ignore duplicate',
                'n:',n,anchor.h,'==>',p.h)
            return
    vc.global_bare_organizer_node_list.append(data)
    if trace:
        anchor,p,n = data
        g.trace('=====',tag,'n:',n,anchor.h,'==>',p.h)
            # '\n  anchor:',anchor.h,
            # '\n  p:',p.h)
#@+node:ekr.20150312225028.71: *7* vc.add_to_pending
def add_to_pending(self,active,p):
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace(active.p.h,'==>',p.h)
    vc.pending.append((active,p.copy()),)
#@+node:ekr.20150312225028.72: *7* vc.add_to_work_list
def add_to_work_list(self,data,tag):
    '''Append the data to the work list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.work_list.append(data)
    if trace:
        active,p = data
        g.trace('=====',tag,active.h,'==>',p.h)
#@+node:ekr.20150312225028.73: *7* vc.anchor_decr
def anchor_decr(self,anchor,p): # p is only for traces.
    '''
    Decrement the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n - 1
    if trace: g.trace(n-1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.74: *7* vc.anchor_incr
def anchor_incr(self,anchor,p): # p is only for traces.
    '''
    Increment the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n + 1
    if trace: g.trace(n+1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.75: *7* vc.clear_pending
def clear_pending(self,active,p):
    '''Clear the appropriate entries from the pending list.'''
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace('===== clear pending',len(vc.pending))
    if False: # active and active.parent_od:
        for data in vc.pending:
            data = active.parent_od.p,data[1]
            vc.add_to_work_list(data,'clear-pending-to-active')
    vc.pending = []
#@+node:ekr.20150312225028.76: *7* vc.enter_organizers
def enter_organizers(self,od,p):
    '''Enter all organizers whose anchors are p.'''
    vc = self
    ods = []
    while od:
        ods.append(od)
        od = od.parent_od
    if ods:
        for od in reversed(ods):
            vc.add_organizer_node(od,p)
#@+node:ekr.20150312225028.77: *7* vc.find_organizer
def find_organizer(self,parent,p):
    '''Return the organizer that organizers p, if any.'''
    trace = False # and not g.unitTesting
    vc = self
    anchor = parent
    ods = vc.anchors_d.get(anchor,[])
    for od in ods:
        if p in od.organized_nodes:
            if trace: g.trace('found:',od.h,'for',p.h)
            return od
    return None
#@+node:ekr.20150312225028.78: *7* vc.terminate_organizers
def terminate_organizers(self,active,p):
    '''Terminate all organizers whose anchors are not ancestors of p.'''
    trace = False # and not g.unitTesting
    od = active
    while od and od.anchor != p and od.anchor.isAncestorOf(p):
        if not od.closed:
            if trace: g.trace('===== closing',od.h)
            od.closed = True
        od = od.parent_od
#@+node:ekr.20150312225028.79: *7* vc.terminate_all_open_organizers
def terminate_all_open_organizers(self):
    '''Terminate all open organizers.'''
    trace = True # and not g.unitTesting
    if 0:
        g.trace()
        for od in self.all_ods:
            if od.opened and not od.closed:
                if trace: g.trace('===== closing',od.h)
                od.closed = True
#@+node:ekr.20150312225028.80: *6* vc.move_nodes & helpers
def move_nodes(self):
    '''Move nodes to their final location and delete the temp node.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.move_nodes_to_organizers(trace)
    vc.move_bare_organizers(trace)
    vc.temp_node.doDelete()
#@+node:ekr.20150312225028.81: *7* vc.move_nodes_to_organizers
def move_nodes_to_organizers(self,trace):
    '''Move all nodes in the work_list.'''
    trace = False # and not g.unitTesting
    trace_dict = False
    trace_moves = False
    trace_deletes = False
    vc = self
    if trace: # A highly useful trace!
        g.trace('\n\nunsorted_list...\n%s' % (
            '\n'.join(['%40s ==> %s' % (parent.h,p.h)
                for parent,p in vc.work_list])))
    # Create a dictionary of each organizers children.
    d = {}
    for parent,p in vc.work_list:
        # This key must remain stable if parent moves.
        key = parent
        aList = d.get(key,[])
        aList.append(p)
        # g.trace(key,[z.h for z in aList])
        d[key] = aList
    if trace and trace_dict:
        # g.trace('d...',sorted([z.h for z in d.keys()]))
        g.trace('d{}...')
        for key in sorted(d.keys()):
            aList = [z.h for z in d.get(key)]
            g.trace('%s %-20s %s' % (id(key),key.h,vc.dump_list(aList,indent=29)))
    # Move *copies* of non-organizer nodes to each organizer.
    organizers = list(d.keys())
    existing_organizers = [z.p.copy() for z in vc.existing_ods]
    moved_existing_organizers = {} # Keys are vnodes, values are positions.
    for parent in organizers:
        aList = d.get(parent,[])
        if trace and trace_moves:
            g.trace('===== moving/copying:',parent.h,
                'with %s children:' % (len(aList)),
                '\n  '+'\n  '.join([z.h for z in aList]))
        for p in aList:
            if p in existing_organizers:
                if trace and trace_moves:
                    g.trace('copying existing organizer:',p.h)
                    g.trace('children:',
                    '\n  '+'\n  '.join([z.h for z in p.children()]))
                copy = vc.copy_tree_to_last_child_of(p,parent)
                old = moved_existing_organizers.get(p.v)
                if old and trace_moves:
                    g.trace('*********** overwrite',p.h)
                moved_existing_organizers[p.v] = copy
            elif p in organizers:
                if trace and trace_moves:
                    g.trace('moving organizer:',p.h)
                aList = d.get(p)
                if aList:
                    if trace and trace_moves: g.trace('**** relocating',
                        p.h,'children:',
                        '\n  '+'\n  '.join([z.h for z in p.children()]))
                    del d[p]
                p.moveToLastChildOf(parent)
                if aList:
                    d[p] = aList
            else:
                parent2 = moved_existing_organizers.get(parent.v)
                if parent2:
                    if trace and trace_moves:
                        g.trace('***** copying to relocated parent:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent2)
                else:
                    if trace and trace_moves: g.trace('copying:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent)
    # Finally, delete all the non-organizer nodes, in reverse outline order.
    def sort_key(od):
        parent,p = od
        return p.sort_key(p)
    sorted_list = sorted(vc.work_list,key=sort_key)
    if trace and trace_deletes:
        g.trace('===== deleting nodes in reverse outline order...')
    for parent,p in reversed(sorted_list):
        if p.v in moved_existing_organizers:
            if trace and trace_deletes:
                g.trace('deleting moved existing organizer:',p.h)
            p.doDelete()
        elif p not in organizers:
            if trace and trace_deletes:
                g.trace('deleting non-organizer:',p.h)
            p.doDelete()
#@+node:ekr.20150312225028.82: *7* vc.move_bare_organizers
def move_bare_organizers(self,trace):
    '''Move all nodes in global_bare_organizer_node_list.'''
    trace = False # and not g.unitTesting
    trace_data = True
    trace_move = True
    vc = self
    # For each parent, sort nodes on n.
    d = {} # Keys are vnodes, values are lists of tuples (n,parent,p)
    existing_organizers = [od.p for od in vc.existing_ods]
    if trace: g.trace('ignoring existing organizers:',
        [p.h for p in existing_organizers])
    for parent,p,n in vc.global_bare_organizer_node_list:
        if p not in existing_organizers:
            key = parent.v
            aList = d.get(key,[])
            if (parent,p,n) not in aList:
                aList.append((parent,p,n),)
                d[key] = aList
    # For each parent, add nodes in childIndex order.
    def key_func(obj):
        return obj[0]
    for key in d.keys():
        aList = d.get(key)
        for data in sorted(aList,key=key_func):
            parent,p,n = data
            n2 = parent.numberOfChildren()
            if trace and trace_data:
                g.trace(n,parent.h,'==>',p.h)
            if trace and trace_move: g.trace(
                'move: %-20s:' % (p.h),
                'to child: %2s' % (n),
                'of: %-20s' % (parent.h),
                'with:',n2,'children')
            p.moveToNthChildOf(parent,n)
#@+node:ekr.20150312225028.83: *7* vc.copy_tree_to_last_child_of
def copy_tree_to_last_child_of(self,p,parent):
    '''Copy p's tree to the last child of parent.'''
    vc = self
    assert p != parent,p
        # A failed assert leads to unbounded recursion.
    # print('copy_tree_to_last_child_of',p.h,parent.h)
    root = parent.insertAsLastChild()
    root.b,root.h = p.b,p.h
    root.v.u = copy.deepcopy(p.v.u)
    for child in p.children():
        vc.copy_tree_to_last_child_of(child,root)
    return root
#@+node:ekr.20150312225028.84: *5* vc.Helpers
#@+node:ekr.20150312225028.85: *6* vc.at_auto_view_body and match_at_auto_body
def at_auto_view_body(self,p):
    '''Return the body text for the @auto-view node for p.'''
    # Note: the unl of p relative to p is simply p.h,
    # so it is pointless to add that to the @auto-view node.
    return 'gnx: %s\n' % p.v.gnx

def match_at_auto_body(self,p,auto_view):
    '''Return True if any line of auto_view.b matches the expected gnx line.'''
    if 0: g.trace(p.b == 'gnx: %s\n' % auto_view.v.gnx,
        g.shortFileName(p.h),auto_view.v.gnx,p.b.strip())
    return p.b == 'gnx: %s\n' % auto_view.v.gnx
#@+node:ekr.20150312225028.86: *6* vc.clean_nodes (not used)
def clean_nodes(self):
    '''Delete @auto-view nodes with no corresponding @auto nodes.'''
    vc = self
    c = vc.c
    views = vc.has_at_views_node()
    if not views:
        return
    # Remember the gnx of all @auto nodes.
    d = {}
    for p in c.all_unique_positions():
        if vc.is_at_auto_node(p):
            d[p.v.gnx] = True
    # Remember all unused @auto-view nodes.
    delete = []
    for child in views.children():
        s = child.b and g.splitlines(child.b)
        gnx = s[len('gnx'):].strip()
        if gnx not in d:
            g.trace(child.h,gnx)
            delete.append(child.copy())
    for p in reversed(delete):
        p.doDelete()
    c.selectPosition(views)
#@+node:ekr.20150312225028.87: *6* vc.comments...
#@+node:ekr.20150312225028.88: *7* vc.comment_delims
def comment_delims(self,p):
    '''Return the comment delimiter in effect at p, an @auto node.'''
    vc = self
    c = vc.c
    d = g.get_directives_dict(p)
    s = d.get('language') or c.target_language
    language,single,start,end = g.set_language(s,0)
    return single,start,end
#@+node:ekr.20150312225028.89: *7* vc.delete_leading_comments
def delete_leading_comments(self,delims,p):
    '''
    Scan for leading comments from p and return them.
    At present, this only works for single-line comments.
    '''
    single,start,end = delims
    if single:
        lines = g.splitLines(p.b)
        result = []
        for s in lines:
            if s.strip().startswith(single):
                result.append(s)
            else: break
        if result:
            p.b = ''.join(lines[len(result):])
            # g.trace('len(result)',len(result),p.h)
            return ''.join(result)
    return None
#@+node:ekr.20150312225028.90: *7* vc.is_comment_node
def is_comment_node(self,p,root,delims=None):
    '''Return True if p.b contains nothing but comments or blank lines.'''
    vc = self
    if not delims:
        delims = vc.comment_delims(root)
    # pylint: disable=unpacking-non-sequence
    single,start,end = delims
    assert single or start and end,'bad delims: %r %r %r' % (single,start,end)
    if single:
        for s in g.splitLines(p.b):
            s = s.strip()
            if s and not s.startswith(single) and not g.isDirective(s):
                return False
        return True
    else:
        def check_comment(s):
            done,in_comment = False,True
            i = s.find(end)
            if i > -1:
                tail = s[i+len(end):].strip()
                if tail: done = True
                else: in_comment = False
            return done,in_comment
        
        done,in_comment = False,False
        for s in g.splitLines(p.b):
            s = s.strip()
            if not s:
                pass
            elif in_comment:
                done,in_comment = check_comment(s)
            elif g.isDirective(s):
                pass
            elif s.startswith(start):
                done,in_comment = check_comment(s[len(start):])
            else:
                # g.trace('fail 1: %r %r %r...\n%s' % (single,start,end,s)
                return False
            if done:
                return False
        # All lines pass.
        return True
#@+node:ekr.20150312225028.91: *7* vc.is_comment_organizer_node
# def is_comment_organizer_node(self,p,root):
    # '''
    # Return True if p is an organizer node in the given @auto tree.
    # '''
    # return p.hasChildren() and vc.is_comment_node(p,root)
#@+node:ekr.20150312225028.92: *7* vc.post_move_comments
def post_move_comments(self,root):
    '''Move comments from the start of nodes to their parent organizer node.'''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    for p in root.subtree():
        if p.hasChildren() and not p.b:
            s = vc.delete_leading_comments(delims,p.firstChild())
            if s:
                p.b = s
                # g.trace(p.h)
#@+node:ekr.20150312225028.93: *7* vc.pre_move_comments
def pre_move_comments(self,root):
    '''
    Move comments from comment nodes to the next node.
    This must be done before any other processing.
    '''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    aList = []
    for p in root.subtree():
        if p.hasNext() and vc.is_comment_node(p,root,delims=delims):
            aList.append(p.copy())
            next = p.next()
            if p.b: next.b = p.b + next.b
    # g.trace([z.h for z in aList])
    c.deletePositionsInList(aList)
        # This sets c.changed.
#@+node:ekr.20150312225028.94: *6* vc.find...
# The find commands create the node if not found.
#@+node:ekr.20150312225028.95: *7* vc.find_absolute_unl_node
def find_absolute_unl_node(self,unl,priority_header=False):
    '''Return a node matching the given absolute unl.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    vc = self
    aList = unl.split('-->')
    if aList:
        first,rest = aList[0],'-->'.join(aList[1:])
        count = 0
        pos = re.findall(pos_pattern,first)
        nth_sib,pos = pos[0] if pos else (0,0)
        pos = int(pos) if pos else 0
        nth_sib = int(nth_sib)
        first = re.sub(pos_pattern,"",first).replace('--%3E','-->')
        for parent in vc.c.rootPosition().self_and_siblings():
            if parent.h.strip() == first.strip():
                if pos == count:
                    if rest:
                        return vc.find_position_for_relative_unl(parent,rest,priority_header=priority_header)
                    else:
                        return parent
                count = count+1
        #Here we could find and return the nth_sib if an exact header match was not found
    return None
#@+node:ekr.20150312225028.96: *7* vc.find_at_auto_view_node & helper
def find_at_auto_view_node (self,root):
    '''
    Return the @auto-view node for root, an @auto node.
    Create the node if it does not exist.
    '''
    vc = self
    views = vc.find_at_views_node()
    p = vc.has_at_auto_view_node(root)
    if not p:
        p = views.insertAsLastChild()
        p.h = '@auto-view:' + root.h[len('@auto'):].strip()
        p.b = vc.at_auto_view_body(root)
    return p
#@+node:ekr.20150312225028.97: *7* vc.find_clones_node
def find_at_clones_node(self,root):
    '''
    Find the @clones node for root, an @auto node.
    Create the @clones node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@clones'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.98: *7* vc.find_at_headlines_node
def find_at_headlines_node(self,root):
    '''
    Find the @headlines node for root, an @auto node.
    Create the @headlines node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@headlines'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.99: *7* vc.find_gnx_node
def find_gnx_node(self,gnx):
    '''Return the first position having the given gnx.'''
    # This is part of the read logic, so newly-imported
    # nodes will never have the given gnx.
    vc = self
    for p in vc.c.all_unique_positions():
        if p.v.gnx == gnx:
            return p
    return None
#@+node:ekr.20150312225028.100: *7* vc.find_organizers_node
def find_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, and @auto node.
    Create the @organizers node if it doesn't exist.
    '''
    vc = self
    c = vc.c
    h = '@organizers'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.101: *7* vc.find_position_for_relative_unl
def find_position_for_relative_unl(self,parent,unl,priority_header=False):
    '''
    Return the node in parent's subtree matching the given unl.
    The unl is relative to the parent position.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    # This is called from finish_create_organizers & compute_all_organized_positions.
    trace = False # and not g.unitTesting
    trace_loop = True
    trace_success = False
    vc = self
    if not unl:
        if trace and trace_success:
            g.trace('return parent for empty unl:',parent.h)
        return parent
    # The new, simpler way: drop components of the unl automatically.
    drop,p = [],parent # for debugging.
    # if trace: g.trace('p:',p.h,'unl:',unl)
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    for s in unl.split('-->'):
        found = False # The last part must match.
        if 1:
            # Create the list of children on the fly.
            aList = vc.headlines_dict.get(p.v)
            if aList is None:
                aList = [z.h for z in p.children()]
                vc.headlines_dict[p.v] = aList
            try:
                pos = re.findall(pos_pattern,s)
                nth_sib,pos = pos[0] if pos else (0,0)
                pos = int(pos) if pos else 0
                nth_sib = int(nth_sib)
                s = re.sub(pos_pattern,"",s).replace('--%3E','-->')
                indices = [i for i, x in enumerate(aList) if x == s]
                if len(indices)>pos:
                    #First we try the nth node with same header
                    n = indices[pos]
                    p = p.nthChild(n)
                    found = True
                elif len(indices)>0:
                    #Then we try any node with same header
                    n = indices[-1]
                    p = p.nthChild(n)
                    found = True
                elif not priority_header:
                    #Then we go for the child index if return_pos is true
                    if len(aList)>nth_sib:
                        n = nth_sib
                    else:
                        n = len(aList)-1
                    if n>-1:
                        p = p.nthChild(n)
                    else:
                        g.es('Partial UNL match: Referenced level is higher than '+str(p.level()))
                    found = True
                if trace and trace_loop: g.trace('match:',s)
            except ValueError: # s not in aList.
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
        else: # old code.
            for child in p.children():
                if child.h == s:
                    p = child
                    found = True
                    if trace and trace_loop: g.trace('match:',s)
                    break
                # elif trace and trace_loop: g.trace('no match:',child.h)
            else:
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
    if not found and priority_header:
        aList = []
        for p in vc.c.all_unique_positions():
            if p.h.replace('--%3E','-->') in unl:
                aList.append((p.copy(),p.get_UNL(False,False,True)))
        unl_list = [re.sub(pos_pattern,"",x).replace('--%3E','-->') for x in unl.split('-->')]
        for iter_unl in aList:
            maxcount = 0
            count = 0
            compare_list = unl_list[:]
            for header in reversed(iter_unl[1].split('-->')):
                if re.sub(pos_pattern,"",header).replace('--%3E','-->') == compare_list[-1]:
                    count = count+1
                    compare_list.pop(-1)
                else:
                    break
            if count > maxcount:
                p = iter_unl[0]
                found = True
    if found:
        if trace and trace_success:
            g.trace('found unl:',unl,'parent:',p.h,'drop',drop)
    else:
        if trace: g.trace('===== unl not found:',unl,'parent:',p.h,'drop',drop)
    return p if found else None
#@+node:ekr.20150312225028.102: *7* vc.find_representative_node
def find_representative_node (self,root,target):
    '''
    root is an @auto node. target is a clones node within root's tree.
    Return a node *outside* of root's tree that is cloned to target,
    preferring nodes outside any @<file> tree.
    Never return any node in any @views or @view tree.
    '''
    trace = False and not g.unitTesting
    assert target
    assert root
    vc = self
    # Pass 1: accept only nodes outside any @file tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p.isAnyAtFileNode():
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 1:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    # Pass 2: accept any node outside the root tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p == root:
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 2:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    g.trace('no representative node for:',target,'parent:',target.parent())
    return None
#@+node:ekr.20150312225028.103: *7* vc.find_views_node
def find_at_views_node(self):
    '''
    Find the first @views node in the outline.
    If it does not exist, create it as the *last* top-level node,
    so that no existing positions become invalid.
    '''
    vc = self
    c = vc.c
    p = g.findNodeAnywhere(c,'@views')
    if not p:
        last = c.rootPosition()
        while last.hasNext():
            last.moveToNext()
        p = last.insertAfter()
        p.h = '@views'
        # c.selectPosition(p)
        # c.redraw()
    return p
#@+node:ekr.20150312225028.104: *6* vc.has...
# The has commands return None if the node does not exist.
#@+node:ekr.20150312225028.105: *7* vc.has_at_auto_view_node
def has_at_auto_view_node(self,root):
    '''
    Return the @auto-view node corresponding to root, an @root node.
    Return None if no such node exists.
    '''
    vc = self
    c = vc.c
    assert vc.is_at_auto_node(root) or vc.is_at_file_node(root),root
    views = g.findNodeAnywhere(c,'@views')
    if views:
        # Find a direct child of views with matching headline and body.
        for p in views.children():
            if vc.match_at_auto_body(p,root):
                return p
    return None
#@+node:ekr.20150312225028.106: *7* vc.has_clones_node
def has_at_clones_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@clones')
#@+node:ekr.20150312225028.107: *7* vc.has_at_headlines_node
def has_at_headlines_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@headlines')
#@+node:ekr.20150312225028.108: *7* vc.has_organizers_node
def has_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, an @auto node.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@organizers')
#@+node:ekr.20150312225028.109: *7* vc.has_views_node
def has_at_views_node(self):
    '''Return the @views or None if it does not exist.'''
    vc = self
    return g.findNodeAnywhere(vc.c,'@views')
#@+node:ekr.20150312225028.110: *6* vc.is...
#@+node:ekr.20150312225028.111: *7* vc.is_at_auto_node
def is_at_auto_node(self,p):
    '''Return True if p is an @auto node.'''
    return g.match_word(p.h,0,'@auto') and not g.match(p.h,0,'@auto-')
        # Does not match @auto-rst, etc.

def is_at_file_node(self,p):
    '''Return True if p is an @file node.'''
    return g.match_word(p.h,0,'@file')
#@+node:ekr.20150312225028.112: *7* vc.is_cloned_outside_parent_tree
def is_cloned_outside_parent_tree(self,p):
    '''Return True if a clone of p exists outside the tree of p.parent().'''
    return len(list(set(p.v.parents))) > 1
#@+node:ekr.20150312225028.113: *7* vc.is_organizer_node
def is_organizer_node(self,p,root):
    '''
    Return True if p is an organizer node in the given @auto tree.
    '''
    vc = self
    return p.hasChildren() and vc.is_comment_node(p,root)

#@+node:ekr.20150312225028.114: *6* vc.testing...
#@+node:ekr.20150312225028.115: *7* vc.compare_test_trees
def compare_test_trees(self,root1,root2):
    '''
    Compare the subtrees whose roots are given.
    This is called only from unit tests.
    '''
    vc = self
    s1,s2 = vc.trial_write(root1),vc.trial_write(root2)
    if s1 == s2:
        return True
    g.trace('Compare:',root1.h,root2.h)
    p2 = root2.copy().moveToThreadNext()
    for p1 in root1.subtree():
        if p1.h == p2.h:
            g.trace('Match:',p1.h)
        else:
            g.trace('Fail: %s != %s' % (p1.h,p2.h))
            break
        p2.moveToThreadNext()
    return False
#@+node:ekr.20150312225028.116: *7* vc.compare_trial_writes
def compare_trial_writes(self,s1,s2):
    '''
    Compare the two strings, the results of trial writes.
    Stop the comparison after the first mismatch.
    '''
    trace_matches = False
    full_compare = False
    lines1,lines2 = g.splitLines(s1),g.splitLines(s2)
    i,n1,n2 = 0,len(lines1),len(lines2)
    while i < n1 and i < n2:
        s1,s2 = lines1[i].rstrip(),lines2[i].rstrip()
        i += 1
        if s1 == s2:
            if trace_matches: g.trace('Match:',s1)
        else:
            g.trace('Fail:  %s != %s' % (s1,s2))
            if not full_compare: return
    if i < n1:
        g.trace('Extra line 1:',lines1[i])
    if i < n2:
        g.trace('Extra line 2:',lines2[i])
#@+node:ekr.20150312225028.117: *7* vc.dump_list
def dump_list(self,aList,indent=4):
    '''Dump a list, one item per line.'''
    lead = '\n' + ' '*indent
    return lead+lead.join(sorted(aList))
#@+node:ekr.20150312225028.118: *7* vc.trial_write
def trial_write(self,root):
    '''
    Return a trial write of outline whose root is given.
    
    **Important**: the @auto import and write code end all nodes with
    newlines. Because no imported nodes are empty, the code below is
    *exactly* equivalent to the @auto write code as far as trailing
    newlines are concerned. Furthermore, we can treat Leo directives as
    ordinary text here.
    '''
    vc = self
    if 1:
        # Do a full trial write, exactly as will be done later.
        at = vc.c.atFileCommands
        ok = at.writeOneAtAutoNode(root,
            toString=True,force=True,trialWrite=True)
        if ok:
            return at.stringOutput
        else:
            g.trace('===== can not happen')
            return ''
    elif 1:
        # Concatenate all body text.  Close, but not exact.
        return ''.join([p.b for p in root.self_and_subtree()])
    else:
        # Compare headlines, ignoring nodes without body text and comment nodes.
        # This was handy during early development.
        return '\n'.join([p.h for p in root.self_and_subtree()
            if p.b and not p.h.startswith('#')])
#@+node:ekr.20150312225028.119: *6* vc.unls...
#@+node:ekr.20150312225028.120: *7* vc.drop_all_organizers_in_unl
def drop_all_organizers_in_unl(self,organizer_unls,unl):
    '''Drop all organizer unl's in unl, recreating the imported unl.'''
    vc = self
    def unl_sort_key(s):
        return s.count('-->')
    for s in reversed(sorted(organizer_unls,key=unl_sort_key)):
        if unl.startswith(s):
            s2 = vc.drop_unl_tail(s)
            unl = s2 + unl[len(s):]
    return unl[3:] if unl.startswith('-->') else unl
#@+node:ekr.20150312225028.121: *7* vc.drop_unl_tail & vc.drop_unl_parent
def drop_unl_tail(self,unl):
    '''Drop the last part of the unl.'''
    return '-->'.join(unl.split('-->')[:-1])

def drop_unl_parent(self,unl):
    '''Drop the penultimate part of the unl.'''
    aList = unl.split('-->')
    return '-->'.join(aList[:-2] + aList[-1:])
#@+node:ekr.20150312225028.122: *7* vc.get_at_organizer_unls
def get_at_organizer_unls(self,p):
    '''Return the unl: lines in an @organizer: node.'''
    return [s[len('unl:'):].strip()
        for s in g.splitLines(p.b)
            if s.startswith('unl:')]

#@+node:ekr.20150312225028.123: *7* vc.relative_unl & unl
def relative_unl(self,p,root):
    '''Return the unl of p relative to the root position.'''
    vc = self
    result = []
    ivar = vc.headline_ivar
    for p in p.self_and_parents():
        if p == root:
            break
        else:
            h = getattr(p.v,ivar,p.h)
            result.append(h)
    return '-->'.join(reversed(result))

def unl(self,p):
    '''Return the unl corresponding to the given position.'''
    vc = self
    return '-->'.join(reversed([
        getattr(p.v,vc.headline_ivar,p.h)
            for p in p.self_and_parents()]))
    # return '-->'.join(reversed([p.h for p in p.self_and_parents()]))
#@+node:ekr.20150312225028.124: *7* vc.source_unl
def source_unl(self,organizer_unls,organizer_unl):
    '''Return the unl of the source node for the given organizer_unl.'''
    vc = self
    return vc.drop_all_organizers_in_unl(organizer_unls,organizer_unl)
#@+node:ekr.20150312225028.125: *7* vc.unl_tail
def unl_tail(self,unl):
    '''Return the last part of a unl.'''
    return unl.split('-->')[:-1][0]
#@+node:ekr.20150312225028.126: *4* vc.Commands
@g.command('view-pack')
def view_pack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.pack()

@g.command('view-unpack')
def view_unpack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.unpack()
        
@g.command('at-file-to-at-auto')
def at_file_to_at_auto_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.convert_at_file_to_at_auto(c.p)
#@+node:ekr.20140711111623.17795: *4* class ConvertController (leoPersistence.py)
class ConvertController(object):
    '''A class to convert @file trees to @auto trees.'''

    def __init__(self, c, p):
        self.c = c
        self.pd = c.persistenceController
        self.root = p.copy()
    @others
#@+node:ekr.20140711111623.17796: *5* convert.delete_at_data_nodes
def delete_at_data_nodes(self, root):
    '''Delete all @data nodes pertaining to root.'''
    cc = self
    pd = cc.pd
    while True:
        p = pd.has_at_data_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20140711111623.17797: *5* convert.import_from_string
def import_from_string(self, s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    # ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    scanner = g.app.scanner_for_ext(c, ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True, c=c, parent=p, s=s)
    p.h = root.h.replace('@file', '@auto' if ok else '@@auto')
    return ok, p
#@+node:ekr.20140711111623.17798: *5* convert.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root, pd = cc.root, c.persistenceController
    # set the expected imported headline for all vnodes.
    t1 = time.time()
    cc.set_expected_imported_headlines(root)
    t2 = time.time()
    # Delete all previous @data nodes for this tree.
    cc.delete_at_data_nodes(root)
    t3 = time.time()
    # Ensure that all nodes of the tree are regularized.
    ok = pd.prepass(root)
    t4 = time.time()
    if not ok:
        g.es_print('Can not convert', root.h, color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
            # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4 - t3),
            '\n  total:                           %4.2f sec' % (t4 - t1))
        return
    # Create the appropriate @data node.
    at_auto_view = pd.update_before_write_foreign_file(root)
    t5 = time.time()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.time()
    if trace and trace_s:
        g.trace('source file...\n', s)
    # Import the @auto string.
    ok, p = cc.import_from_string(s)
    t7 = time.time()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = pd.at_data_body(p)
        # Recreate the organizer nodes, headlines, etc.
        pd.update_after_read_foreign_file(p)
        t8 = time.time()
        # if not ok:
            # p.h = '@@' + p.h
            # g.trace('restoring original @auto file')
            # ok,p = cc.import_from_string(s)
            # if ok:
                # p.h = '@@' + p.h + ' (restored)'
                # if p.next():
                    # p.moveAfter(p.next())
        t9 = time.time()
    else:
        t8 = t9 = time.time()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
        # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4 - t3),
        '\n  update_before_write_foreign_file:%4.2f sec' % (t5 - t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6 - t5),
        '\n  import_from_string:              %4.2f sec' % (t7 - t6),
        '\n  update_after_read_foreign_file   %4.2f sec' % (t8 - t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9 - t8),
        '\n  total:                           %4.2f sec' % (t9 - t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20140711111623.17799: *5* convert.set_expected_imported_headlines
def set_expected_imported_headlines(self, root):
    '''Set v._imported_headline for every vnode.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    aClass = g.app.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic, atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c, root, importing=True)
    fn = c.os_path_finalize_join(theDir, root.h)
    fn = root.h.replace('\\', '/')
    junk, fn = g.os_path_split(fn)
    fn, junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner, 'headlineForNode'):
        for p in root.subtree():
            if not hasattr(p.v, '_imported_headline'):
                h = scanner.headlineForNode(fn, p)
                setattr(p.v, '_imported_headline', h)
                if trace and h != p.h:
                    g.trace('==>', h) # p.h,'==>',h
#@+node:ekr.20140711111623.17800: *5* convert.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind='@file',
        nosentinels=True,
        perfectImportFlag=False,
        scriptWrite=False,
        thinFile=True,
        toString=True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:', ok, 's:...\n' + s)
    return s
#@+node:ekr.20140711111623.17794: *4* pd.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self, root):
    if root.isAtFileNode():
        ConvertController(self.c, root).run()
    else:
        g.es_print('not an @file node:', root.h)
#@+node:ekr.20140131101641.15495: *4* pd.prepass & helper
def prepass(self, root):
    '''Make sure root's tree has no hard-to-handle nodes.'''
    c, pd = self.c, self
    ic = c.importCommands
    ic.tab_width = c.getTabWidth(root)
    language = g.scanForAtLanguage(c, root)
    ext = g.app.language_extension_dict.get(language)
    if not ext: return
    if not ext.startswith('.'): ext = '.' + ext
    scanner = g.app.scanner_for_ext(c, ext)
    if not scanner:
        g.trace('no scanner for', root.h)
        return True # Pretend all went well.
    # Pass 1: determine the nodes to be inserted.
    ok = True
    # parts_list = []
    for p in root.subtree():
        ok2 = pd.regularize_node(p, scanner)
        ok = ok and ok2
    return ok
#@+node:ekr.20140131101641.15496: *5* pd.regularize_node
def regularize_node(self, p, scanner):
    '''Regularize node p so that it will not cause problems.'''
    c = self.c
    ok = scanner(atAuto=True, c=c, parent=p, s=p.b)
        # The scanner is a callback returned by g.app.scanner_for_ext.
        # It must have a c argument.
    if not ok:
        g.es_print('please regularize:', p.h)
    return ok
#@+node:ekr.20150312225028.6: *3* class LogManager (not used yet)
class LogManager:

    '''A class to handle the global log, and especially
    switching the log from commander to commander.'''

    def __init__ (self):

        trace = (False or g.trace_startup) and not g.unitTesting
        if trace: g.es_debug('(LogManager)')

        self.log = None             # The LeoFrame containing the present log.
        self.logInited = False      # False: all log message go to logWaiting list.
        self.logIsLocked = False    # True: no changes to log are allowed.
        self.logWaiting = []        # List of messages waiting to go to a log.
        self.printWaiting = []      # Queue of messages to be sent to the printer.
        self.signon_printed = False # True: the global signon has been printed.

    @others
#@+node:ekr.20150312225028.7: *4* LogM.setLog, lockLog, unlocklog
def setLog (self,log):

    """set the frame to which log messages will go"""

    # print("app.setLog:",log,g.callers())
    if not self.logIsLocked:
        self.log = log

def lockLog(self):
    """Disable changes to the log"""
    self.logIsLocked = True

def unlockLog(self):
    """Enable changes to the log"""
    self.logIsLocked = False
#@+node:ekr.20150312225028.8: *4* LogM.writeWaitingLog
def writeWaitingLog (self,c):
    '''Write all waiting lines to the log.'''
    trace = True
    lm = self
    if trace:
        # Do not call g.es, g.es_print, g.pr or g.trace here!
        print('** writeWaitingLog','silent',g.app.silentMode,c.shortFileName())
        # print('writeWaitingLog',g.callers())
        # import sys ; print('writeWaitingLog: argv',sys.argv)
    if not c or not c.exists:
        return
    if g.unitTesting:
        lm.printWaiting = []
        lm.logWaiting = []
        g.app.setLog(None) # Prepare to requeue for other commanders.
        return
    table = [
        ('Leo Log Window','red'),
        (g.app.signon,'black'),
        (g.app.signon2,'black'),
    ]
    table.reverse()
    c.setLog()
    lm.logInited = True # Prevent recursive call.
    if not lm.signon_printed:
        lm.signon_printed = True
        if not g.app.silentMode:
            print('')
            print('** isPython3: %s' % g.isPython3)
            if not g.enableDB:
                print('** caching disabled')
            print(g.app.signon)
            print(g.app.signon2)
    if not g.app.silentMode:
        for s in lm.printWaiting:
            print(s)
    lm.printWaiting = []
    if not g.app.silentMode:
        for s,color in table:
            lm.logWaiting.insert(0,(s+'\n',color),)
        for s,color in lm.logWaiting:
            g.es('',s,color=color,newline=0)
                # The caller must write the newlines.
    lm.logWaiting = []
    # Essential when opening multiple files...
    lm.setLog(None)
#@+node:ekr.20200209055601.1: **  Archived test stuff
#@+node:ekr.20200209055610.1: *3* @@button test
g.cls()
import os
leo_editor_dir = os.path.join(g.app.loadDir, '..', '..')
os.chdir(leo_editor_dir)
if 0: # Run all tests
    commands = f"python -m unittest leo.core.leoAst"
    g.execute_shell_commands(commands, trace=False)
else:
    file_ = '.leoAst'
    class_, test = '', ''
    class_, test = '.TestOrange', '.test_join_leading_whitespace'
    # class_, test = '.TestOrange', '.test_leo_sentinels'
    # class_, test = '.TestOrange', '.test_start_of_line_whitespace'
    # class_, test = '.TestOrange', '.test_join_lines'
    #
    # Older
    #
    # file_ = '.leoAst'
    # class_, test = '.TestTOG', ''
    # class_, test = '.TestFiles', '.compare_tog_vs_asttokens'
    # class_, test = '.TestFiles', '.optional_file_tests'
    # class_, test = '.TestTokens', '.show_asttokens_script'
    # class_, test = '.TestFstringify', '.test_call_with_comments'
        # # 'test_call_in_rhs' # '.test_braces'
    # class_, test = '.TestOrange','.test_join_lines'
        # # 'test_sync_tokens' # '.test_multi_line_pet_peeves'
    # class_, test = '.TestTokens', '.show_example_dump'
        # # 'test_line_links'
    # class_, test = '.TestTopLevelFunctions', '.test_get_encoding_directive'
    # class_, test = '.TestReassignTokens', ''
    # class_, test = '.TestTOT', ''
    commands = f"python -m unittest leo.core{file_}{class_}{test}"
    g.execute_shell_commands(commands, trace=False)
#@+node:ekr.20200209055640.1: *3* @@button pt
# Run pytest coverage tests.
g.cls()
import os
leo_editor_dir = os.path.join(g.app.loadDir, '..', '..')
os.chdir(leo_editor_dir)
commands = "pytest --cov-report html --cov-report term-missing --cov leo.core.leoAst leo/core/leoAst.py"
g.execute_shell_commands(commands, trace=False)
#@+node:ekr.20200219073857.1: *3* @@suite run all doctests
tm = c.testManager
path = g.os_path_join(g.app.loadDir,"..","core")
exclude = ['leoIPython.py',]
    # leoIPython.py will give an annoying error if IPython can't be imported.
modules = tm.importAllModulesInPath(path,exclude=exclude)
suite   = tm.createUnitTestsFromDoctests(modules)
#@+node:ekr.20200505050955.1: *3* ----- mvc-prototype branch
@language rest
@wrap

- checkout the mvc-prototype branch

- open leo/core/LeoPyRef.leo in Leo and save this document as leo/core/LeoPyRef.db
  This should provide a large outline for the prototype to play with.

- python leo/extensions/myleoqt.py leo\core\leoPy.db
#@+node:vitalije.20200502083732.1: *4* @@@file ../extensions/myleoqt.py
#@+node:vitalije.20200502091628.1: *4* <<imports>>
from collections import defaultdict
import sys
import os
import pickle
from PyQt5 import QtCore, QtGui, QtWidgets
assert QtGui
import sqlite3
import time
import timeit
import unittest
from hypothesis.strategies import lists, integers, sampled_from, data
from hypothesis import given, settings
from datetime import timedelta
import random

# Set the path before importing Leo files.
LEO_INSTALLED_AT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
if LEO_INSTALLED_AT not in sys.path:
    sys.path.append(LEO_INSTALLED_AT)
LEO_ICONS_DIR = os.path.join(LEO_INSTALLED_AT, 'leo', 'themes', 'light', 'Icons')
try:
    import leo.core.leoNodes as leoNodes
    import leo.core.leoGlobals as g
except ImportError:
    print('===== import error!')
g.app = g.bunch(nodeIndices=leoNodes.NodeIndices('vitalije'))

Q = QtCore.Qt
#@+node:vitalije.20200502103535.1: *4* draw_tree
def draw_tree(tree, root, icons):
    def additem(par, parItem):
        for ch in par.children:
            item = QtWidgets.QTreeWidgetItem()
            item.setFlags(item.flags() |
                          Q.ItemIsEditable |
                          item.DontShowIndicatorWhenChildless)
            parItem.addChild(item)
            item.setText(0, ch.h)
            item.setData(0, 1024, ch)
            item.setExpanded(ch.isExpanded())
            item.setIcon(0, icons[ch.computeIcon()])
            additem(ch, item)
    ritem = tree.invisibleRootItem()
    ritem.setData(0, 1024, root)
    additem(root, ritem)
#@+node:vitalije.20200508091419.1: *4* execute_script
def execute_script(self):
    c = self.c
    t = self.tree
    root = t.invisibleRootItem()
    curr = t.currentItem()
    currpath = path_to_item(root, curr)

    # now we need to replace current top level items
    # with the their clones so that we can preserve
    # the original ones for undo
    t.blockSignals(True)
    n = root.childCount()
    olditems = [root.takeChild(0) for x in range(n)]
    for item in olditems:
        root.addChild(item.clone())

    # let's find current item in the new tree
    ncurr = item_from_path(root, currpath)
    t.setCurrentItem(ncurr)
    t.blockSignals(False)

    # store old v-nodes data
    old_vs = set((x, x.h, x.b) for x in c.fileCommands.gnxDict.values())

    # new values for the redo operation
    # if the script finishes without exception
    # these values will be updated after the script finishes
    # for now they are the same as the old values
    newitems = olditems
    new_vs = old_vs
    newcurr = curr

    # some more data for the locals in script
    p = QtPosition(ncurr)
    v = p.v
    script = v.b

    @others

    try:
        exec(script, dict(c=c, v=v, p=p, tree=t, root=root))
        # update new values and redraw the tree
        nset = set((x, x.h, x.b) for x in c.fileCommands.gnxDict.values())
        new_vs =  nset - old_vs
        old_vs = old_vs - nset
        item = t.currentItem()
        newv = item.data(0, 1024)
        npath = path_to_item(root, item)
        self.redraw()

        newcurr = item_from_path(root, npath)
        # we want to keep the same node selected if possible
        # but script may delete previously selected node or
        # some other node so their position won't match
        if not newcurr or newcurr.data(0, 1024) != newv:
            # let's try to find item that points to the newv
            for item in iter_all_v_items(root, newv):
                newcurr = item
                break

        # remember the newitems for redo
        n2 = root.childCount()
        newitems = [root.child(i) for i in range(n2)]

        # select node if we found one to select
        if newcurr:
            t.setCurrentItem(newcurr)

    except KeyboardInterrupt:
        # sometimes script got into an infinite loop
        # or work too long
        undoexec()
        return
    except:
        g.es_print_exception()
        undoexec()
        return
    # script terminated successfully let's add undo/redo pair
    self.c.addUndo(undoexec, redoexec)
#@+node:vitalije.20200508104717.1: *5* undoexec
def undoexec():
    for v, h, b in old_vs:
        v.b = b
        v.h = h
    t.blockSignals(True)
    root.takeChildren()
    for item in olditems:
        root.addChild(item)
    reconnect_vnodes(root)
    t.blockSignals(False)
    t.setCurrentItem(curr)
#@+node:vitalije.20200508104721.1: *5* redoexec
def redoexec():
    for v, h, b in new_vs:
        v.b = b
        v.h = h
    t.blockSignals(True)
    root.takeChildren()
    for item in newitems:
        root.addChild(item)
    reconnect_vnodes(root)
    t.blockSignals(False)
    t.setCurrentItem(newcurr)
#@+node:vitalije.20200507182943.1: *4* test_select_and_commnads
@settings(max_examples=50, deadline=timedelta(seconds=4))
@given(data())
def test_select_and_commnads(data):
    app = demo2_app()

    index = data.draw(integers(min_value=0, max_value=10000))
    names = data.draw(lists(sampled_from(method_names), min_size=5,
                             max_size=100))

    select_item_at_index(app, index)
    a1, b1 = outline_shapes(app)
    for name in names:
        meth = getattr(app, name)
        meth()
        app.processEvents() #
        if method_names.index(name) > 3:
            # possible change in the outline
            a2, b2 = outline_shapes(app)
            assert a2 == b2
            if a1 != a2:
                # the change is real - let's undo it
                app.undo()
                a, b = outline_shapes(app)
                assert a == a1
                assert b == b1
                app.redo()
            a1, b1 = a2, b2
#@+node:ekr.20200303071111.1: ** ----- Macros
# Most of this code dates from Leo 5.5.
#@+node:ekr.20200303061541.113: *3* 5.0 COPY ga.get_arg (entry) & helpers
def get_arg (ga,event,
    returnKind=None,returnState=None,handler=None,
    tabList=[],completion=True,oneCharacter=False,
    stroke=None,useMinibuffer=True
):
    << ga.get_arg docstring >>
    # pylint: disable=unpacking-non-sequence
    trace = False and not g.app.unitTesting
    c,k = ga.c,ga.k
    state = k.getState('getArg')
    c.check_event(event)
    # Remember these events also.
    if c.macroCommands.recordingMacro and state > 0:
        c.macroCommands.startRecordingMacro(event)
    char = event and event.char or ''
    if state > 0:
        k.setLossage(char,stroke)
    if state == 0:
        ga.do_state_zero(completion,event,handler,oneCharacter,
            returnKind,returnState,tabList,useMinibuffer)
        if trace: ga.trace_state(char,completion,handler,state,stroke)
    else:
        if trace: ga.trace_state(char,completion,handler,state,stroke)
        if char == 'Escape':
            k.keyboardQuit()
        elif ga.should_end(char,stroke):
            ga.do_end(event,char,stroke)
        elif char in('\t','Tab'):
            if trace: g.trace('***tab***')
            ga.do_tab(ga.tabList,ga.arg_completion)
        elif char in ('\b','BackSpace'):
            ga.do_back_space(ga.tabList,ga.arg_completion)
            c.minibufferWantsFocus()
        elif k.isFKey(stroke):
            # Ignore only F-keys. Ignoring all except plain keys would kill unicode searches.
            pass
        else:
            ga.do_char(event,char)
#@+node:ekr.20200303061541.114: *4* << ga.get_arg docstring >>
'''
Accumulate an argument. Enter the given return state when done.

Ctrl-G will abort this processing at any time.

All commands needing user input call k.getArg, which just calls ga.get_arg.

The arguments to ga.get_arg are as follows:
    
event:              The event passed to the command.
    
returnKind=None:    A string.
returnState=None,   An int.
handler=None,       A function.

    When the argument is complete, ga.do_end does::

        if kind: k.setState(kind,n,handler)

tabList=[]:         A list of possible completions.
      
completion=True:    True if completions are enabled.

oneCharacter=False: True if k.arg should be a single character.

stroke=None:        The incoming key stroke.

useMinibuffer=True: True: put focus in the minibuffer while accumulating arguments.
                    False allows sort-lines, for example, to show the selection range.

'''
#@+node:ekr.20200303061541.115: *4* ga.do_char
def do_char (ga,event,char):
    '''Handle a non-special character.'''
    k = ga.k
    k.updateLabel(event)
    # Any plain key resets tab cycling.
    ga.reset_tab_cycling()
#@+node:ekr.20200303061541.116: *4* ga.do_end
def do_end(ga,event,char,stroke):
    '''A return or escape has been seen.'''
    trace = False and not g.unitTesting
    c,k = ga.c,ga.k
    if trace:
        g.trace('char',repr(char),stroke,k.getArgEscapes)
        if ga.after_get_arg_state:
            kind,n,handler = ga.after_get_arg_state
            g.trace('after state',kind,n,handler and handler.__name__ or 'None')
    if char == '\t' and char in k.getArgEscapes:
        k.getArgEscapeFlag = True
    if stroke and stroke in k.getArgEscapes:
        k.getArgEscapeFlag = True
    if k.oneCharacterArg:
        k.arg = char
    else:
        k.arg = ga.get_label()
        if trace: g.trace('k.mb_prefix',k.mb_prefix,'k.arg',k.arg)
    kind,n,handler = ga.after_get_arg_state
    if kind: k.setState(kind,n,handler)
    ga.log.deleteTab('Completion')
    if trace: g.trace('kind',kind,'n',n,'handler',handler and handler.__name__)
    # pylint: disable=not-callable
    ga.reset_tab_cycling()
    if handler: handler(event)
#@+node:ekr.20200303061541.117: *4* ga.do_state_zero
def do_state_zero(ga,completion,event,handler,oneCharacter,
    returnKind,returnState,tabList,useMinibuffer
):
    '''Do state 0 processing.'''
    trace = False and not g.unitTesting
    c,k = ga.c,ga.k
    # Set the ga globals...
    ga.after_get_arg_state=returnKind,returnState,handler
    ga.arg_completion = completion
    ga.cycling_prefix = None
    ga.handler = handler
    ga.tabList = tabList[:] if tabList else []
    # Set the k globals...
    k.argSelectedText = c.frame.body.wrapper.getSelectedText()
    k.functionTail = None
    k.oneCharacterArg = oneCharacter
    # Do *not* change the label here!
    # Enter the next state.
    c.widgetWantsFocus(c.frame.body.wrapper)
    k.setState('getArg',1,k.getArg)
    k.afterArgWidget = event and event.widget or c.frame.body.wrapper
    if useMinibuffer: c.minibufferWantsFocus()
#@+node:ekr.20200303061541.118: *4* ga.should_end
def should_end(ga,char,stroke):
    '''Return True if ga.get_arg should return.'''
    k = ga.k
    return (
        char in ('\n','Return',) or
        k.oneCharacterArg or 
        stroke and stroke in k.getArgEscapes or
        char == '\t' and char in k.getArgEscapes
            # The Find Easter Egg.
    )
#@+node:ekr.20200303061541.119: *4* ga.trace_state
def trace_state(ga,char,completion,handler,state,stroke):
    '''Trace the vars and ivars.'''
    k = ga.c.k
    g.trace(
        'state',state,'char',repr(char),'stroke',repr(stroke),
        # 'isPlain',k.isPlainKey(stroke),
        '\n',
        'escapes',k.getArgEscapes,
        'completion',ga.arg_completion,
        'handler',ga.handler and ga.handler.__name__ or 'None',
    )
#@+node:ekr.20200303061541.120: *3* 5.0 COPY k.fullCommand (alt-x) & helper
def fullCommand (self,event,specialStroke=None,specialFunc=None,help=False,helpHandler=None):
    '''Handle 'full-command' (alt-x) mode.'''
    trace = False and not g.unitTesting
    verbose = False
    k = self ; c = k.c
    recording = c.macroCommands.recordingMacro
    state = k.getState('full-command')
    helpPrompt = 'Help for command: '
    c.check_event(event)
    ch = char = event and event.char or ''
    stroke = event and event.stroke or None
    if trace and verbose: g.trace(g.callers())
    if trace:
        g.trace('recording',recording,'state',state,char)
    if recording:
        c.macroCommands.startRecordingMacro(event)
    if state > 0:
        k.setLossage(char,stroke)
    if state == 0:
        k.mb_event = event # Save the full event for later.
        k.setState('full-command',1,handler=k.fullCommand)
        prompt = helpPrompt if help else k.altX_prompt
        k.setLabelBlue(prompt)
        k.mb_help = help
        k.mb_helpHandler = helpHandler
        c.minibufferWantsFocus()
    elif char == 'Ins' or k.isFKey(char):
        pass
    elif char == 'Escape':
        k.keyboardQuit()
    elif char in ('Down','Up'):
        # Do command history.
        h,i = k.commandHistory,k.commandIndex
        if h:
            i = min(i+1,len(h)-1) if char == 'Up' else max(0,i-1)
            commandName = h[i]
            k.commandIndex = i
            # g.trace(char,i,h,h[i])
            k.setLabel(k.mb_prefix + commandName)
    elif char in ('\n','Return'):
        if trace and verbose: g.trace('***Return')
        c.frame.log.deleteTab('Completion')
        if k.mb_help:
            s = k.getLabel()
            commandName = s[len(helpPrompt):].strip()
            k.clearState()
            k.resetLabel()
            if k.mb_helpHandler: k.mb_helpHandler(commandName)
        else:
            s = k.getLabel(ignorePrompt=True)
            commandName = s.strip()
            ok = k.callAltXFunction(k.mb_event)
            if ok:
                # Update command history if the command exists.
                h,i = k.commandHistory,k.commandIndex
                if commandName in h:
                    h.remove(commandName)
                    k.commandIndex = max(0,i-1)
                h.append(commandName)
    elif char in ('\t','Tab'):
        if trace and verbose: g.trace('***Tab')
        k.doTabCompletion(list(c.commandsDict.keys()))
        c.minibufferWantsFocus()
    elif char in ('\b','BackSpace'):
        if trace and verbose: g.trace('***BackSpace')
        k.doBackSpace(list(c.commandsDict.keys()))
        c.minibufferWantsFocus()
    elif k.ignore_unbound_non_ascii_keys and len(ch) > 1:
        # g.trace('non-ascii')
        if specialStroke:
            g.trace(specialStroke)
            specialFunc()
        c.minibufferWantsFocus()
    else:
        # Clear the list, any other character besides tab indicates that a new prefix is in effect.
        k.mb_tabList = []
        k.updateLabel(event)
        k.mb_tabListPrefix = k.getLabel()
        c.minibufferWantsFocus()
        # g.trace('new prefix',k.mb_tabListPrefix)
#@+node:ekr.20200303061541.121: *4* callAltXFunction
def callAltXFunction (self,event):
    '''Call the function whose name is in the minibuffer.'''
    trace = False and not g.unitTesting
    k = self ; c = k.c ; s = k.getLabel()
    k.mb_tabList = []
    commandName,tail = k.getMinibufferCommandName()
    if trace: g.trace('command:',commandName,'tail:',tail)
    k.functionTail = tail
    func = c.commandsDict.get(commandName)
    k.newMinibufferWidget = None
    # g.trace(func and func.__name__,'mb_event',event and event.widget.widgetName)
    if func:
        # These must be done *after* getting the command.
        k.clearState()
        k.resetLabel()
        if commandName != 'repeat-complex-command':
            k.mb_history.insert(0,commandName)
        w = event and event.widget
        if hasattr(w,'permanent') and not w.permanent:
            # In a headline that is being edited.
            ### g.es('Can not execute commands from headlines')
            c.endEditing()
            c.bodyWantsFocusNow()
            # Change the event widget so we don't refer to the to-be-deleted headline widget.
            event.w = event.widget = c.frame.body.wrapper.widget
            func(event)
        else:
            c.widgetWantsFocusNow(event and event.widget) # Important, so cut-text works, e.g.
            func(event)
        k.endCommand(commandName)
        return True
    else:
        # Show possible completions if the command does not exist.
        if 1: # Useful.
            if trace: g.trace('*** tab completion')
            k.doTabCompletion(list(c.commandsDict.keys()))
        else: # Annoying.
            k.keyboardQuit()
            k.setStatusLabel('Command does not exist: %s' % commandName)
            c.bodyWantsFocus()
        return False
#@+node:ekr.20050710142719: *3* COPY 5.5: leoEditCommands.py
@first # -*- coding: utf-8 -*-
'''
Basic editor commands for Leo.

Modelled after Emacs and Vim commands.
'''
# pylint: disable=eval-used
<< imports >>
<< define class BaseEditCommandsClass >>
@others
#@+node:ekr.20050710151017: *4* << imports >> (leoEditCommands)
import leo.core.leoGlobals as g
# import leo.core.LeoFind as LeoFind
import difflib   
docutils = g.importExtension('docutils',pluginName='leoEditCommands.py')
try:
    import enchant
except ImportError:
    enchant = None
import os
import re
if g.isPython3:
    from functools import reduce
import shlex
import string
import subprocess # Always exists in Python 2.6 and above.
import sys
#@+node:ekr.20050920084036.1: *4* << define class BaseEditCommandsClass >>
class BaseEditCommandsClass:

    '''The base class for all edit command classes'''

    @others
#@+node:ekr.20050920084036.2: *5*  Birth (BaseEditCommandsClass)
def __init__ (self,c):

    self.c = c
    self.k = None
    self.registers = {} # To keep pychecker happy.
    self.undoData = None
    self.w = None

def finishCreate(self):

    self.k = self.c.k
    try:
        self.w = self.c.frame.body.wrapper # New in 4.4a4.
    except AttributeError:
        self.w = None

def init (self):
    '''Called from k.keyboardQuit to init all classes.'''
    pass
#@+node:ekr.20051214132256: *5* begin/endCommand (baseEditCommands)
#@+node:ekr.20051214133130: *6* beginCommand  & beginCommandWithEvent
def beginCommand (self,undoType='Typing'):

    '''Do the common processing at the start of each command.'''

    return self.beginCommandHelper(ch='',undoType=undoType,w=self.w)

def beginCommandWithEvent (self,event,undoType='Typing'):

    '''Do the common processing at the start of each command.'''

    return self.beginCommandHelper(
        ch=event and event.char or '',
        undoType=undoType,w=event.widget)
#@+node:ekr.20051215102349: *7* beingCommandHelper
# New in Leo 4.4b4: calling beginCommand is valid for all widgets,
# but does nothing unless we are in the body pane.

def beginCommandHelper (self,ch,undoType,w):

    c = self.c ; p = c.p
    name = c.widget_name(w)

    if name.startswith('body'):
        oldSel =  w.getSelectionRange()
        oldText = p.b
        self.undoData = b = g.Bunch()
        # To keep pylint happy.
        b.ch=ch
        b.name=name
        b.oldSel=oldSel
        b.oldText=oldText
        b.w=w
        b.undoType=undoType
    else:
        self.undoData = None

    return w
#@+node:ekr.20051214133130.1: *6* endCommand (baseEditCommand)
# New in Leo 4.4b4: calling endCommand is valid for all widgets,
# but handles undo only if we are in body pane.

def endCommand(self,label=None,changed=True,setLabel=True):

    '''Do the common processing at the end of each command.'''

    trace =  False and not g.unitTesting
    c = self.c ; b = self.undoData ; k = self.k

    if trace: g.trace('changed',changed)

    if b and b.name.startswith('body') and changed:
        c.frame.body.onBodyChanged(undoType=b.undoType,
            oldSel=b.oldSel,oldText=b.oldText,oldYview=None)

    self.undoData = None # Bug fix: 1/6/06 (after a5 released).

    k.clearState()

    # Warning: basic editing commands **must not** set the label.
    if setLabel:
        if label:
            k.setLabelGrey(label)
        else:
            k.resetLabel()
#@+node:ekr.20061007105001: *5* editWidget (BaseEditCommandsClass)
def editWidget (self,event,forceFocus=True):
    '''Return the edit widget for the event.'''
    trace = False and not g.unitTesting
    c = self.c
    w = event and event.widget
    wname = (w and c.widget_name(w)) or '<no widget>'
    isTextWrapper = g.isTextWrapper(w)
    # New in Leo 4.5: single-line editing commands apply to minibuffer widget.
    if w and isTextWrapper:
        self.w = w
    else:
        self.w = self.c.frame.body and self.c.frame.body.wrapper
    if trace: g.trace(isTextWrapper,wname,w)
    if self.w and forceFocus:
        c.widgetWantsFocusNow(self.w)
    return self.w
#@+node:ekr.20050920084036.5: *5* getPublicCommands & getStateCommands
def getPublicCommands (self):

    '''Return a dict describing public commands implemented in the subclass.
    Keys are untranslated command names.  Values are methods of the subclass.'''

    return {}
#@+node:ekr.20050920084036.6: *5* getWSString
def getWSString (self,s):

    return ''.join([ch if ch=='\t' else ' ' for ch in s])
#@+node:ekr.20050920084036.7: *5* oops
def oops (self):

    g.pr("BaseEditCommandsClass oops:",
        g.callers(),
        "must be overridden in subclass")
#@+node:ekr.20050929161635: *5* Helpers
#@+node:ekr.20050920084036.249: *6* _chckSel
def _chckSel (self,event,warning='no selection'):

    k = self.k
    w = self.editWidget(event)
    val = w and w.hasSelection()
    if warning and not val:
        k.setLabelGrey(warning)
    return val
#@+node:ekr.20050920084036.250: *6* _checkIfRectangle
def _checkIfRectangle (self,event):

    c,k = self.c,self.k

    key = event and event.char.lower() or ''

    val = self.registers.get(key)

    if val and type(val) == type([]):
        k.clearState()
        k.setLabelGrey("Register contains Rectangle, not text")
        return True

    return False
#@+node:ekr.20050920084036.233: *6* getRectanglePoints
def getRectanglePoints (self,w):

    c = self.c
    c.widgetWantsFocusNow(w)

    s = w.getAllText()
    i,j = w.getSelectionRange()
    r1,r2 = g.convertPythonIndexToRowCol(s,i)
    r3,r4 = g.convertPythonIndexToRowCol(s,j)

    return r1+1,r2,r3+1,r4
#@+node:ekr.20051002090441: *6* keyboardQuit
def keyboardQuit (self,event=None):

    '''Clear the state and the minibuffer label.'''

    return self.k.keyboardQuit()
#@+node:ekr.20120315062642.9746: *4*  Commands defined by @g.command
#@+node:ekr.20120315062642.9745: *5* ctrl-click-at-cursor
@g.command('ctrl-click-at-cursor')
def ctrlClickAtCursor(event):
    c = event.get('c')
    if c:
        g.openUrlOnClick(event)
#@+node:ekr.20140701101218.16740: *5* enable/disable/toggle-idle-time-events
@g.command('disable-idle-time-events')
def disable_idle_time_events(event):
    g.disableIdleTimeHook()
    
@g.command('enable-idle-time-events')
def enable_idle_time_events(event):
    g.enableIdleTimeHook()
    
@g.command('toggle-idle-time-events')
def toggle_idle_time_events(event):
    if g.app.idleTimeHook:
        g.disableIdleTimeHook()
    else:
        g.enableIdleTimeHook()
#@+node:ekr.20140701093157.16738: *5* enable/disable/toggle-idle-time-tracing
@g.command('disable-idle-time-tracing')
def disable_idle_time_tracing(event):
    g.app.trace_idle_time = False
    
@g.command('enable-idle-time-tracing')
def enable_idle_time_tracing(event):
    g.app.trace_idle_time = True
    
@g.command('toggle-idle-time-tracing')
def toggle_idle_time_tracing(event):
    g.app.trace_idle_time = not g.app.trace_idle_time
#@+node:ekr.20131120115900.16323: *5* join-leo-irc
@g.command('join-leo-irc')
def join_leo_irc(event=None):
    import webbrowser
    webbrowser.open("http://webchat.freenode.net/?channels=%23leo&uio=d4")
#@+node:ekr.20120315062642.9742: *5* open-url
@g.command('open-url')
def openUrl(event=None):
    c = event.get('c')
    if c:
        g.openUrl(c.p)

#@+node:ekr.20131120115900.16324: *5* open-url-under-cursor
@g.command('open-url-under-cursor')
def openUrlUnderCursor(event=None):
    return g.openUrlOnClick(event)
#@+node:ekr.20140718105559.17735: *5* pylint command
@g.command('pylint')
def pylint_command(event):
    << class PylintCommand >>
    c = event.get('c')
    if c:
        PylintCommand(c).run()
#@+node:ekr.20140718105559.17736: *6* << class PylintCommand >>
class PylintCommand:
    '''A class to run pylint on all Python @<file> nodes in c.p's tree.'''
    def __init__(self,c):
        '''ctor for PylintCommand class.'''
        self.c = c
        self.seen = [] # List of checked vnodes.
        self.wait = True
            # The no-wait code doesn't seem to work.
    @others
#@+node:ekr.20140718114031.17742: *7* check
def check(self,p,rc_fn):
    '''Check a single node.  Return True if it is a Python @<file> node.'''
    found = False
    if p.isAnyAtFileNode():
        # Fix bug: https://github.com/leo-editor/leo-editor/issues/67
        aList = g.get_directives_dict_list(p)
        path = c.scanAtPathDirectives(aList)
        fn = p.anyAtFileNodeName()
        if fn.endswith('.py'):
            fn = g.os_path_finalize_join(path,fn)
            if p.v not in self.seen:
                self.seen.append(p.v)
                self.run_pylint(fn,rc_fn)
                found = True
    return found
#@+node:ekr.20140719051145.17727: *7* get_rc_file
def get_rc_file(self):
    '''Return the path to the pylint configuration file.'''
    trace = False and not g.unitTesting
    base = 'pylint-leo-rc.txt'
    table = (
        g.os_path_finalize_join(g.app.homeDir,'.leo',base),
            # In ~/.leo
        g.os_path_finalize_join(g.app.loadDir,'..','..','leo','test',base),
            # In leo/test
    )
    for fn in table:
        fn = g.os_path_abspath(fn)
        if g.os_path_exists(fn):
            if trace: g.trace('found:',fn)
            return fn
    g.es_print('no pylint configuration file found in\n%s' % (
        '\n'.join(table)))
    return None
#@+node:ekr.20140718105559.17737: *7* run
def run(self):
    '''Run Pylint on all Python @<file> nodes in c.p's tree.'''
    c,root = self.c,self.c.p
    try:
        import time
        from pylint import lint
        # in pythonN/Lib/site-packages.
    except ImportError:
        g.warning('can not import pylint')
        return
    rc_fn = self.get_rc_file()
    if not rc_fn:
        return
    # Run lint on all Python @<file> nodes in root's tree.
    t1 = time.clock()
    found = False
    for p in root.self_and_subtree():
        found |= self.check(p,rc_fn)
    # Look up the tree if no @<file> nodes were found.
    if not found:
        for p in root.parents():
            if self.check(p,rc_fn):
                found = True
                break
    # If still not found, expand the search if root is a clone.
    if not found:
        isCloned = any([p.isCloned() for p in root.self_and_parents()])
        # g.trace(isCloned,root.h)
        if isCloned:
            for p in c.all_positions():
                if p.isAnyAtFileNode():
                    isAncestor = any([z.v == root.v for z in p.self_and_subtree()])
                    # g.trace(isAncestor,p.h)
                    if isAncestor and self.check(p,rc_fn):
                        break
    if self.wait:
        g.es_print('pylint: done %s' % g.timeSince(t1))
#@+node:ekr.20140718105559.17741: *7* run_pylint
def run_pylint(self,fn,rc_fn):
    '''Run pylint on fn with the given pylint configuration file.'''
    if not os.path.exists(fn):
        print('file not found:',fn)
        return
    # Report the file name.
    if self.wait:
        print('pylint: %s' % (g.shortFileName(fn)))
    # Create the required args.
    args = ','.join([
        "fn=r'%s'" % (fn),
        "rc=r'%s'" % (rc_fn),
    ])
    # Execute the command in a separate process.
    command = '%s -c "import leo.core.leoGlobals as g; g.run_pylint(%s)"' % (
        sys.executable,args)
    if not self.wait:
        command = '&' + command
    g.execute_shell_commands(command)   
#@+node:ekr.20120211121736.10817: *4* class EditCommandsManager
class EditCommandsManager:

    '''A class to init all edit commands properly.

    This class eliminates the circular dependencies that otherwise
    would arise from the module-level classesList.
    '''


    @others
#@+node:ekr.20120211121736.10830: *5*  ecm.ctor
def __init__ (self,c):

    self.c = c

    self.classesList = (
        ('abbrevCommands',      AbbrevCommandsClass),
        ('bufferCommands',      BufferCommandsClass),
        ('editCommands',        EditCommandsClass),
        ('chapterCommands',     ChapterCommandsClass),
        ('controlCommands',     ControlCommandsClass),
        ('debugCommands',       DebugCommandsClass),
        ('editFileCommands',    EditFileCommandsClass),
        ('helpCommands',        HelpCommandsClass),
        ('keyHandlerCommands',  KeyHandlerCommandsClass),
        ('killBufferCommands',  KillBufferCommandsClass),
        ('leoCommands',         LeoCommandsClass),
        ('macroCommands',       MacroCommandsClass),
        # ('queryReplaceCommands',QueryReplaceCommandsClass),
        ('recTangleCommands',   RecTangleCommandsClass),
        ('registerCommands',    RegisterCommandsClass),
        ('searchCommands',      SearchCommandsClass),
        ('spellCommands',       SpellCommandsClass),
    )


#@+node:ekr.20120211121736.10827: *5* ecm.createEditCommanders
def createEditCommanders (self):

    '''Create edit classes in the commander.'''

    c = self.c

    for name, theClass in self.classesList:
        theInstance = theClass(c)# Create the class.
        setattr(c,name,theInstance)
        # g.trace(name,theInstance)
#@+node:ekr.20120211121736.10828: *5* ecm.getPublicCommands
def getPublicCommands (self):
    '''Add the names of commands defined in this file to c.commandsDict.'''
    c,d = self.c,{}
    for name, theClass in self.classesList:
        theInstance = getattr(c,name)
        theInstance.finishCreate()
        theInstance.init()
        d2 = theInstance.getPublicCommands()
        if d2:
            d.update(d2)
            if 0:
                g.pr('----- %s' % name)
                for key in sorted(d2): g.pr(key)
    c.commandsDict.update(d)
#@+node:ekr.20120211121736.10829: *5* ecm.initAllEditCommanders
def initAllEditCommanders (self):

    '''Re-init classes in the commander.'''

    c = self.c

    for name, theClass in self.classesList:
        theInstance = getattr(c,name)
        theInstance.init()
#@+node:ekr.20050920084036.13: *4* abbrevCommandsClass
class AbbrevCommandsClass (BaseEditCommandsClass):

    '''A class to handle user-defined abbreviations.

    See apropos-abbreviations for details.'''

    @others
#@+node:ekr.20100901080826.6002: *5*  Birth
#@+node:ekr.20100901080826.6003: *6* ctor (abbrevCommandsClass
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c)
        # init the base class.
    # Set local ivars.
    self.abbrevs = {} # Keys are names, values are (abbrev,tag).
    self.daRanges = []
    self.dynaregex = re.compile( # For dynamic abbreviations
        r'[%s%s\-_]+'%(string.ascii_letters,string.digits))
        # Not a unicode problem.
    self.event = None
    self.globalDynamicAbbrevs = c.config.getBool('globalDynamicAbbrevs')
    self.last_hit = None # Distinguish between text and tree abbreviations.
    self.store ={'rlist':[], 'stext':''} # For dynamic expansion.
    self.tree_abbrevs_d = {} # Keys are names, values are (tree,tag).
    self.w = None
#@+node:ekr.20100901080826.6004: *6* finishCreate (abbrevClass) & helpers
def finishCreate(self):
    
    c,k = self.c,self.c.k
    BaseEditCommandsClass.finishCreate(self)
    self.init_settings()
    self.init_abbrev()
    self.init_tree_abbrev()
    self.init_env()
    if 0: # Annoying.
        if (not g.app.initing and not g.unitTesting and
            not g.app.batchMode and not c.gui.isNullGui
        ):
            g.red('Abbreviations %s' % ('on' if c.k.abbrevOn else 'off'))
#@+node:ekr.20130924110246.13738: *7* init_abbrev
def init_abbrev(self):
    '''
    Init the user abbreviations from @data global-abbreviations
    and @data abbreviations nodes.
    '''
    c = self.c
    table = (
        ('global-abbreviations','global'),
        ('abbreviations','local'),
    )
    for source,tag in table:
        aList = c.config.getData(source,strip_data=False) or []
        abbrev,result = [],[]
        for s in aList:
            if s.startswith('\\:'):
                # Continue the previous abbreviation.
                abbrev.append(s[2:])
            else:
                # End the previous abbreviation.
                if abbrev:
                    result.append(''.join(abbrev))
                    abbrev = []
                # Start the new abbreviation.
                if s.strip():
                    abbrev.append(s)
        # End any remaining abbreviation.
        if abbrev:
            result.append(''.join(abbrev))
        for s in result:
            self.addAbbrevHelper(s,tag)
#@+node:ekr.20130924110246.13748: *7* init_env
def init_env(self):
    '''
    Init c.abbrev_subst_env by executing the contents of the
    @data abbreviations-subst-env node.
    '''
    
    c = self.c
    if c.abbrev_place_start and self.enabled:
        aList = self.subst_env
        result = []
        for z in aList:
            # Compatibility with original design.
            if z.startswith('\\:'):
                result.append(z[2:])
            else:
                result.append(z)
        result = ''.join(result)
        try:
            exec(result,c.abbrev_subst_env,c.abbrev_subst_env)
        except Exception:
            g.es('Error exec\'ing @data abbreviations-subst-env')
            g.es_exception()
    else:
        c.abbrev_subst_start = False
        # if c.config.getString('abbreviations-subst-start'):
            # g.es("Note: @abbreviations-subst-start found, but no substitutions "
                 # "without @scripting-at-script-nodes = True")
#@+node:ekr.20130924110246.13749: *7* init_settings
def init_settings(self):
    
    c = self.c
    c.k.abbrevOn = c.config.getBool('enable-abbreviations',default=False)
    # Init these here for k.masterCommand.
    c.abbrev_place_end = c.config.getString('abbreviations-place-end')
    c.abbrev_place_start = c.config.getString('abbreviations-place-start')
    c.abbrev_subst_end = c.config.getString('abbreviations-subst-end')
    c.abbrev_subst_env = {'c': c,'g': g,'_values': {},}
        # The environment for all substitutions.
        # May be augmented in init_env.
    c.abbrev_subst_start = c.config.getString('abbreviations-subst-start')
    # Local settings.
    self.enabled = (
        c.config.getBool('scripting-at-script-nodes') or
        c.config.getBool('scripting-abbreviations'))
    self.subst_env = c.config.getData('abbreviations-subst-env',strip_data=False)
    
#@+node:ekr.20131113150347.17255: *7* init_tree_abbrev
def init_tree_abbrev (self):
    '''Init tree_abbrevs_d from @data tree-abbreviations nodes.'''
    trace = False ; verbose = True
    c = self.c
    fn = c.shortFileName()
    d = {} # Keys are abbreviation names; Values are (xml) strings.
    # Careful. This happens early in startup.
    root = c.rootPosition()
    if not root:
        # if trace and verbose: g.trace('no root',fn)
        return
    if not c.p:
        c.selectPosition(root)
    if not c.p:
        if trace and verbose: g.trace('no c.p',fn)
        return
    tree_s = c.config.getOutlineData('tree-abbreviations')
    if not tree_s:
        if trace and verbose: g.trace('no tree_s',fn)
        return
    if trace and verbose: g.trace(fn,len(tree_s or ''))
    # Expand the tree so we can traverse it.
    if not c.canPasteOutline(tree_s):
        if trace and verbose: g.trace('bad copied outline',fn)
        return
    c.fileCommands.leo_file_encoding='utf-8'
    old_p = c.p.copy()
    p = c.pasteOutline(s=tree_s,redrawFlag=False,undoFlag=False,tempOutline=True)
    if not p: return g.trace('no pasted node')
    for s in g.splitLines(p.b):
        if s.strip() and not s.startswith('#'):
            abbrev_name = s.strip()
            for child in p.children():
                if child.h.strip() == abbrev_name:
                    c.selectPosition(child)
                    abbrev_s = c.fileCommands.putLeoOutline()
                    if trace and verbose:
                        g.trace('define',abbrev_name,len(abbrev_s))
                        # g.trace('define',abbrev_name,'\n\n',abbrev_s)
                    d[abbrev_name] = abbrev_s
                    break
            else:
                g.trace('no definition for %s' % abbrev_name)
    p.doDelete(newNode=old_p)
    c.selectPosition(old_p)
    if trace and (d or verbose):
        if verbose:
            g.trace(fn)
            for key in sorted(d.keys()):
                g.trace(key,'...\n\n',d.get(key))
        else:
            g.trace(fn,sorted(d.keys()))
    self.tree_abbrevs_d = d
#@+node:ekr.20050920084036.15: *6* getPublicCommands & getStateCommands
def getPublicCommands (self):
    '''Return a dict: keys are command names and whose values are methods.'''
    return {
        # Reload.
        # 'reload-abbreviations': self.reloadAbbreviations,
        # Non-prefixed commands.
        'toggle-abbrev-mode':   self.toggleAbbrevMode,
        # Dynamic...
        'dabbrev-completion':   self.dynamicCompletion,
        'dabbrev-expands':      self.dynamicExpansion,
        # Static...
        'abbrev-add-global':        self.addAbbreviation,
        # 'abbrev-expand-region':   self.regionalExpandAbbrev,
        'abbrev-inverse-add-global':self.addInverseAbbreviation,
        'abbrev-kill-all':          self.killAllAbbrevs,
        'abbrev-list':              self.listAbbrevs,
        'abbrev-read':              self.readAbbreviations,
        'abbrev-write':             self.writeAbbreviations,
    }
#@+node:ekr.20140104063158.17234: *6* reloadAbbreviations
def reloadAbbreviations(self):
    '''Reload all abbreviations from all files.'''
#@+node:ekr.20050920084036.27: *5* expandAbbrev & helpers (entry point)
def expandAbbrev (self,event,stroke):
    '''
    Not a command.  Called from k.masterCommand to expand
    abbreviations in event.widget.

    Words start with '@'.
    '''
    trace = False and not g.unitTesting
    verbose = False
    c = self.c
    ch = event and event.char or ''
    w = self.editWidget(event,forceFocus=False)
    if not w: return False
    if w.hasSelection(): return False
    assert g.isStrokeOrNone(stroke),stroke
    if stroke in ('BackSpace','Delete'):
        if trace and verbose: g.trace(stroke)
        return False
    d = {'Return':'\n','Tab':'\t','space':' ','underscore':'_'}
    if stroke:
        ch = d.get(stroke.s,stroke.s)
        if len(ch) > 1:
            if (stroke.find('Ctrl+') > -1 or
                stroke.find('Alt+') > -1 or
                stroke.find('Meta+') > -1
            ):
                ch = ''
            else:
                ch = event and event.char or ''
    else:
        ch = event.char
    if trace and verbose: g.trace('ch',repr(ch),'stroke',repr(stroke))
    # New code allows *any* sequence longer than 1 to be an abbreviation.
    # Any whitespace stops the search.
    s = w.getAllText()
    j = w.getInsertPoint()
    i = j-1
    while len(s) > i >= 0 and s[i] not in ' \t\n':
        prefix = s[i:j]
        word = prefix+ch
        val,tag = self.tree_abbrevs_d.get(word),'tree'
        # if val: g.trace('*****',word,'...\n\n',len(val))
        if not val:
            val,tag = self.abbrevs.get(word,(None,None))
        if val:
            if trace and verbose: g.trace(repr(word),'val',val,'tag',tag)
            # Require a word match if the abbreviation is itself a word.
            if ch in ' \t\n': word = word.rstrip()
            if word.isalnum() and word[0].isalpha():
                if i == 0 or s[i-1] in ' \t\n':
                    break
                else:
                    i -= 1
            else:
                break
        else: i -= 1
    else:
        return False
    c.abbrev_subst_env['_abr'] = word
    if tag == 'tree':
        self.last_hit = c.p.copy()
        self.expand_tree(w,i,j,val,word)
    else:
        # Never expand a search for text matches.
        place_holder = '__NEXT_PLACEHOLDER' in val
        if place_holder:
            expand_search = bool(self.last_hit)
        else:
            self.last_hit = None
            expand_search = False
        if trace: g.trace('expand_search',expand_search,'last_hit',self.last_hit)
        self.expand_text(w,i,j,val,word,expand_search)
    c.frame.body.forceFullRecolor()
    c.bodyWantsFocusNow()
    return True
#@+node:ekr.20131113150347.17257: *6* expand_text
def expand_text(self,w,i,j,val,word,expand_search=False):
    '''Make a text expansion at location i,j of widget w.'''
    c = self.c
    # g.trace(i,j,word,g.callers())
    val,do_placeholder = self.make_script_substitutions(i,j,val)
    self.replace_abbrev_name(w,i,j,val)
    # Search to the end.  We may have been called via a tree abbrev.
    p = c.p.copy()
    if expand_search:
        while p:
            if self.find_place_holder(p,do_placeholder):
                return
            else:
                p.moveToThreadNext()
    else:
        self.find_place_holder(p,do_placeholder)
#@+node:ekr.20131113150347.17258: *6* expand_tree & helper
def expand_tree(self,w,i,j,tree_s,word):
    '''Paste tree_s as children of c.p.'''
    c,u = self.c,self.c.undoer
    if not c.canPasteOutline(tree_s):
        return g.trace('bad copied outline: %s' % tree_s)
    old_p = c.p.copy()
    bunch = u.beforeChangeTree(old_p)
    self.replace_abbrev_name(w,i,j,None)
    self.paste_tree(old_p,tree_s)
    # Make all script substitutions first.
    for p in old_p.subtree():
        # Search for the next place-holder.
        val,do_placeholder = self.make_script_substitutions(0,0,p.b)
        if not do_placeholder: p.b = val
    # Now search for all place-holders.
    for p in old_p.subtree():
        if self.find_place_holder(p,do_placeholder):
            break
    u.afterChangeTree(old_p,'tree-abbreviation',bunch)
#@+node:ekr.20131114051702.22732: *6* find_place_holder
def find_place_holder(self,p,do_placeholder):
    '''
    Search for the next place-holder.
    If found, select the place-holder (without the delims).
    '''
    c = self.c
    s = p.b
    if do_placeholder or c.abbrev_place_start and c.abbrev_place_start in s:
        new_s,i,j = self.next_place(s,offset=0)
        if i is None:
            return False
        w = c.frame.body.wrapper
        switch = p != c.p
        if switch:
            c.selectPosition(p)
        else:
            scroll = w.getYScrollPosition()
        oldSel = w.getSelectionRange()
        w.setAllText(new_s)
        c.frame.body.onBodyChanged(undoType='Typing',oldSel=oldSel)
        c.p.b = new_s
        if switch:
            c.redraw()
        w.setSelectionRange(i,j,insert=j)
        if switch:
            w.seeInsertPoint()
        else:
            # Keep the scroll point if possible.
            w.setYScrollPosition(scroll)
            w.seeInsertPoint()
        return True
    else:
        return False
#@+node:ekr.20131114051702.22731: *6* make_script_substitutions
def make_script_substitutions(self,i,j,val):
    '''Make scripting substitutions in node p.'''
    trace = False and not g.unitTesting
    c = self.c
    if not c.abbrev_subst_start:
        if trace: g.trace('no subst_start')
        return val,False
    # Nothing to undo.
    if c.abbrev_subst_start not in val:
        return val,False
    # Perform all scripting substitutions.
    while c.abbrev_subst_start in val:
        prefix,rest = val.split(c.abbrev_subst_start,1)
        content = rest.split(c.abbrev_subst_end,1)
        if len(content) != 2:
            break
        content,rest = content
        if trace: g.trace('**content',content)
        exec(content,c.abbrev_subst_env,c.abbrev_subst_env)
        val = "%s%s%s" % (prefix,c.abbrev_subst_env['x'],rest)
    if val == "__NEXT_PLACEHOLDER":
        # user explicitly called for next placeholder in an abbrev.
        # inserted previously
        val = ''
        do_placeholder = True
    else:
        do_placeholder = False
        # Huh?
        oldSel = i,j
        c.frame.body.onBodyChanged(undoType='Typing',oldSel=oldSel)
    if trace: g.trace(do_placeholder,val)
    return val,do_placeholder
#@+node:tbrown.20130326094709.25669: *6* next_place
def next_place(self,s,offset=0):
    """
    Given string s containing a placeholder like <| block |>,
    return (s2,start,end) where s2 is s without the <| and |>,
    and start, end are the positions of the beginning and end of block.
    """
    trace = False
    c = self.c
    new_pos = s.find(c.abbrev_place_start,offset)
    new_end = s.find(c.abbrev_place_end,offset)
    if (new_pos < 0 or new_end < 0) and offset:
        new_pos = s.find(c.abbrev_place_start)
        new_end = s.find(c.abbrev_place_end)
        if not(new_pos < 0 or new_end < 0):
            g.es("Found placeholder earlier in body")
    if new_pos < 0 or new_end < 0:
        if trace: g.trace('new_pos',new_pos,'new_end',new_end)
        return s,None,None
    start = new_pos
    place_holder_delim = s[new_pos:new_end+len(c.abbrev_place_end)]
    place_holder = place_holder_delim[
        len(c.abbrev_place_start):-len(c.abbrev_place_end)]
    s2 = s[:start]+place_holder+s[start+len(place_holder_delim):]
    end = start+len(place_holder)
    if trace: g.trace(start,end,g.callers())
    return s2,start,end
#@+node:ekr.20131114124839.16666: *6* paste_tree
def paste_tree(self,old_p,s):
    '''Paste the tree corresponding to s (xml) into the tree.'''
    c = self.c
    c.fileCommands.leo_file_encoding='utf-8'
    p = c.pasteOutline(s=s,redrawFlag=False,undoFlag=False)
    if p:
        # Promote the name node, then delete it.
        p.moveToLastChildOf(old_p)
        c.selectPosition(p)
        c.promote(undoFlag=False)
        p.doDelete()
    else:
        g.trace('paste failed')
#@+node:ekr.20131114124839.17398: *6* replace_abbrev_name
def replace_abbrev_name(self,w,i,j,s):
    '''Replace the abbreviation name by s.'''
    c = self.c
    if i != j:
        w.delete(i,j)
    if s is not None:
        w.insert(i,s)
    oldSel = j,j
    c.frame.body.onBodyChanged(undoType='Abbreviation',oldSel=oldSel)
#@+node:ekr.20050920084036.58: *5* dynamic abbreviation...
#@+node:ekr.20050920084036.60: *6* dynamicCompletion C-M-/
def dynamicCompletion (self,event=None):

    '''dabbrev-completion
    Insert the common prefix of all dynamic abbrev's matching the present word.
    This corresponds to C-M-/ in Emacs.'''

    c,p,u = self.c,self.c.p,self.c.p.v.u
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    if 0 < ins < len(s) and not g.isWordChar(s[ins]): ins -= 1
    i,j = g.getWord(s,ins)
    word = w.get(i,j)
    aList = self.getDynamicList(w,word)
    if not aList: return
    # Bug fix: remove s itself, otherwise we can not extend beyond it.
    if word in aList and len(aList) > 1: aList.remove(word)
    prefix = reduce(g.longestCommonPrefix,aList)
    if prefix.strip():
        ypos = w.getYScrollPosition()
        b = c.undoer.beforeChangeNodeContents(p,oldYScroll=ypos)
        p.b = p.b[:i] + prefix + p.b[j:]
        w.setAllText(p.b)
        w.setInsertPoint(i+len(prefix))
        w.setYScrollPosition(ypos)
        c.undoer.afterChangeNodeContents(p,
            command='dabbrev-completion',bunch=b,dirtyVnodeList=[]) 
#@+node:ekr.20050920084036.59: *6* dynamicExpansion M-/
def dynamicExpansion (self,event=None):

    '''dabbrev-expands (M-/ in Emacs).
    Inserts the longest common prefix of the word at the cursor. Displays
    all possible completions if the prefix is the same as the word.
    '''

    c = self.c
    p = c.p
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    if 0 < ins < len(s) and not g.isWordChar(s[ins]): ins -= 1
    i,j = g.getWord(s,ins)
    word = w.get(i,j)
    aList = self.getDynamicList(w,word)
    if not aList: return
    if word in aList and len(aList) > 1: aList.remove(word)
    prefix = reduce(g.longestCommonPrefix,aList)
    prefix = prefix.strip()
    # g.trace(word,prefix,aList)
    if False and prefix and prefix != word and len(aList) == 1:
        s = w.getAllText()
        ypos = w.getYScrollPosition()
        b = c.undoer.beforeChangeNodeContents(p,oldYScroll=ypos)
        p.b = p.b[:i] + prefix + p.b[j:]
        w.setAllText(p.b)
        w.setInsertPoint(i+len(prefix))
        w.setYScrollPosition(ypos)
        c.undoer.afterChangeNodeContents(p,
            command='dabbrev-expands',bunch=b,dirtyVnodeList=[])
    else:
        self.dynamicExpandHelper(event,prefix,aList,w)
#@+node:ekr.20070605110441: *7* dynamicExpandHelper (added event arg)
def dynamicExpandHelper (self,event,prefix=None,aList=None,w=None):

    c = self.c ; k = c.k ; p = c.p
    tag = 'dabbrev-expand'
    state = k.getState(tag)
    if state == 0:
        self.w = w
        prefix2 = 'dabbrev-expand: '
        c.frame.log.deleteTab('Completion')
        g.es('','\n'.join(aList),tabName='Completion')
        k.setLabelBlue(prefix2+prefix)
        k.getArg(event,tag,1,self.dynamicExpandHelper,tabList=aList)
    else:
        c.frame.log.deleteTab('Completion')
        k.clearState()
        k.resetLabel()
        if k.arg:
            w = self.w
            s = w.getAllText()
            ypos = w.getYScrollPosition()
            b = c.undoer.beforeChangeNodeContents(p,oldYScroll=ypos)
            ins = w.getInsertPoint()
            if 0 < ins < len(s) and not g.isWordChar(s[ins]): ins -= 1
            i,j = g.getWord(s,ins)
            p.b = p.b[:i] + k.arg + p.b[j:]
            w.setAllText(p.b)
            w.setInsertPoint(i+len(k.arg))
            w.setYScrollPosition(ypos)
            c.undoer.afterChangeNodeContents(p,
                command=tag,bunch=b,dirtyVnodeList=[]) 
#@+node:ekr.20050920084036.61: *6* getDynamicList (helper)
def getDynamicList (self,w,s):

    if self.globalDynamicAbbrevs:
        # Look in all nodes.h
        items = []
        for p in self.c.all_unique_positions():
            items.extend(self.dynaregex.findall(p.b))
    else:
        # Just look in this node.
        items = self.dynaregex.findall(w.getAllText())

    items = sorted(set([z for z in items if z.startswith(s)]))

    # g.trace(repr(s),repr(sorted(items)))
    return items
#@+node:ekr.20070531103114: *5* static abbrevs
#@+node:ekr.20100901080826.6001: *6* addAbbrevHelper
def addAbbrevHelper (self,s,tag=''):

    '''Enter the abbreviation 's' into the self.abbrevs dict.'''

    if not s.strip(): return

    try:
        d = self.abbrevs
        data = s.split('=')
        # name = data[0].strip()
        # 2012/02/29: Do *not* strip ws, and allow the user to specify ws.
        name = data[0].replace('\\t','\t').replace('\\n','\n')
        val = '='.join(data[1:])
        if val.endswith('\n'): val = val[:-1]
        val = val.replace('\\n','\n')
        old,tag = d.get(name,(None,None),)
        if old and old != val and not g.unitTesting:
            g.es_print('redefining abbreviation',name,
                '\nfrom',repr(old),'to',repr(val))
        d [name] = val,tag

    except ValueError:
        g.es_print('bad abbreviation: %s' % s)
#@+node:ekr.20050920084036.25: *6* addAbbreviation
def addAbbreviation (self,event):

    '''Add an abbreviation:
    The selected text is the abbreviation;
    the minibuffer prompts you for the name of the abbreviation.
    Also sets abbreviations on.'''

    k = self.k ; state = k.getState('add-abbr')

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        k.setLabelBlue('Add Abbreviation: ')
        k.getArg(event,'add-abbr',1,self.addAbbreviation)
    else:
        w = self.w
        k.clearState()
        k.resetLabel()
        value = k.argSelectedText # 2010/09/01.
        if k.arg.strip():
            self.abbrevs [k.arg] = value,'dynamic'
            k.abbrevOn = True
            k.setLabelGrey(
                "Abbreviation (on): '%s' = '%s'" % (
                    k.arg,value))
#@+node:ekr.20051004080550: *6* addInverseAbbreviation
def addInverseAbbreviation (self,event):

    '''Add an inverse abbreviation:
    The selected text is the abbreviation name;
    the minibuffer prompts you for the value of the abbreviation.'''

    k = self.k ; state = k.getState('add-inverse-abbr')

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        k.setLabelBlue('Add Inverse Abbreviation: ')
        k.getArg(event,'add-inverse-abbr',1,self.addInverseAbbreviation)
    else:
        w = self.w
        k.clearState()
        k.resetLabel()
        s = w.getAllText()
        i = w.getInsertPoint()
        i,j = g.getWord(s,i-1)
        word = s[i:j]
        if word:
            self.abbrevs [word] = k.arg,'add-inverse-abbr'
#@+node:ekr.20050920084036.18: *6* killAllAbbrevs
def killAllAbbrevs (self,event):

    '''Delete all abbreviations.'''

    self.abbrevs = {}
#@+node:ekr.20050920084036.19: *6* listAbbrevs
def listAbbrevs (self,event=None):

    '''List all abbreviations.'''

    d = self.abbrevs
    if d:
        g.es('Abbreviations...')
        keys = list(d.keys())
        keys.sort()
        for name in keys:
            val,tag = d.get(name)
            val = val.replace('\n','\\n')
            tag = tag or ''
            tag = tag+': ' if tag else ''
            g.es('','%s%s=%s' % (tag,name,val))
    else:
        g.es('No present abbreviations')
#@+node:ekr.20050920084036.20: *6* readAbbreviations & helper
def readAbbreviations (self,event=None):

    '''Read abbreviations from a file.'''

    fileName = g.app.gui.runOpenFileDialog(
        title = 'Open Abbreviation File',
        filetypes = [("Text","*.txt"), ("All files","*")],
        defaultextension = ".txt")

    if fileName:
        self.readAbbreviationsFromFile(fileName)
#@+node:ekr.20100901080826.6156: *7* readAbbreviationsFromFile
def readAbbreviationsFromFile(self,fileName):

    k = self.c.k

    try:
        f = open(fileName)
        for s in f:
            self.addAbbrevHelper(s,'file')
        f.close()
        k.abbrevOn = True
        g.es("Abbreviations on")
        # self.listAbbrevs()
    except IOError:
        g.es('can not open',fileName)
#@+node:ekr.20050920084036.23: *6* toggleAbbrevMode
def toggleAbbrevMode (self,event=None):

    '''Toggle abbreviation mode.'''

    k = self.k
    k.abbrevOn = not k.abbrevOn
    k.keyboardQuit()
    if not g.unitTesting and not g.app.batchMode:
        g.es('Abbreviations are ' + 'on' if k.abbrevOn else 'off')
#@+node:ekr.20050920084036.24: *6* writeAbbreviation
def writeAbbreviations (self,event):
    '''Write abbreviations to a file.'''
    fileName = g.app.gui.runSaveFileDialog(
        initialfile = None,
        title='Write Abbreviations',
        filetypes = [("Text","*.txt"), ("All files","*")],
        defaultextension = ".txt")
    if not fileName: return
    try:
        d = self.abbrevs
        f = open(fileName,'w')
        for name in sorted(d.keys()):
            val,tag = self.abbrevs.get(name)
            val=val.replace('\n','\\n')
            s = '%s=%s\n' % (name,val)
            if not g.isPython3:
                s = g.toEncodedString(s,reportErrors=True)
            f.write(s)
        f.close()
        g.es_print('wrote: %s' % fileName)
    except IOError:
        g.es('can not create',fileName)
#@+node:ekr.20050920084036.31: *4* BufferCommandsClass
@
An Emacs instance does not have knowledge of what is considered a
buffer in the environment.

The call to setBufferInteractionMethods calls the buffer configuration methods.
@c

class BufferCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20050920084036.32: *5*  ctor (BufferCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.fromName = '' # Saved name from getBufferName.
    self.nameList = [] # [n: <headline>]
    self.names = {}
    self.tnodes = {} # Keys are n: <headline>, values are tnodes.

    try:
        self.w = c.frame.body.wrapper
    except AttributeError:
        self.w = None
#@+node:ekr.20050920084036.33: *5*  getPublicCommands
def getPublicCommands (self):

    return {

        # These do not seem useful.
            # 'copy-to-buffer':     self.copyToBuffer,
            # 'insert-to-buffer':   self.insertToBuffer,

        'buffer-append-to':             self.appendToBuffer,
        'buffer-kill' :                 self.killBuffer,
        'buffer-prepend-to':            self.prependToBuffer,
        # 'buffer-rename':              self.renameBuffer,
        'buffer-switch-to':             self.switchToBuffer,
        'buffers-list' :                self.listBuffers,
        'buffers-list-alphabetically':  self.listBuffersAlphabetically,
    }
#@+node:ekr.20050920084036.34: *5* Entry points
#@+node:ekr.20050920084036.35: *6* appendToBuffer
def appendToBuffer (self,event):

    '''Add the selected body text to the end of the body text of a named buffer (node).'''

    w = self.editWidget(event) # Sets self.w
    if w:
        self.k.setLabelBlue('Append to buffer: ')
        self.getBufferName(event,self.appendToBufferFinisher)

def appendToBufferFinisher (self,name):

    c,w = self.c,self.w
    s = w.getSelectedText()
    p = self.findBuffer(name)
    if s and p:
        w = self.w
        c.selectPosition(p)
        self.beginCommand('append-to-buffer: %s' % p.h)
        w.insert('end',s)
        w.setInsertPoint('end')
        w.seeInsertPoint()
        self.endCommand()
        c.redraw_after_icons_changed()
        c.recolor()
#@+node:ekr.20050920084036.36: *6* copyToBuffer
def copyToBuffer (self,event):

    '''Add the selected body text to the end of the body text of a named buffer (node).'''

    w = self.editWidget(event) # Sets self.w
    if w:
        self.k.setLabelBlue('Copy to buffer: ')
        self.getBufferName(event,self.copyToBufferFinisher)

def copyToBufferFinisher (self,name):

    c,w = self.c,self.w
    s = w.getSelectedText()
    p = self.findBuffer(name)
    if s and p:
        c.selectPosition(p)
        self.beginCommand('copy-to-buffer: %s' % p.h)
        w.insert('end',s)
        w.setInsertPoint('end')
        self.endCommand()
        c.redraw_after_icons_changed()
        c.recolor()
#@+node:ekr.20050920084036.37: *6* insertToBuffer
def insertToBuffer (self,event):

    '''Add the selected body text at the insert point of the body text of a named buffer (node).'''

    w = self.editWidget(event) # Sets self.w
    if w:
        self.k.setLabelBlue('Insert to buffer: ')
        self.getBufferName(event,self.insertToBufferFinisher)

def insertToBufferFinisher (self,name):

    c,w = self.c,self.w
    s = w.getSelectedText()
    p = self.findBuffer(name)
    if s and p:
        c.selectPosition(p)
        self.beginCommand('insert-to-buffer: %s' % p.h)
        i = w.getInsertPoint()
        w.insert(i,s)
        w.seeInsertPoint()
        self.endCommand()
        c.redraw_after_icons_changed()
#@+node:ekr.20050920084036.38: *6* killBuffer
def killBuffer (self,event):

    '''Delete a buffer (node) and all its descendants.'''

    w = self.editWidget(event) # Sets self.w
    if not w: return

    self.k.setLabelBlue('Kill buffer: ')
    self.getBufferName(event,self.killBufferFinisher)

def killBufferFinisher (self,name):

    c = self.c ; p = self.findBuffer(name)
    if p:
        h = p.h
        current = c.p
        c.selectPosition(p)
        c.deleteOutline (op_name='kill-buffer: %s' % h)
        c.selectPosition(current)
        self.k.setLabelBlue('Killed buffer: %s' % h)
        c.redraw(current)
#@+node:ekr.20050920084036.42: *6* listBuffers & listBuffersAlphabetically
def listBuffers (self,event):

    '''List all buffers (node headlines), in outline order.
    Nodes with the same headline are disambiguated by giving their parent or child index.
    '''

    self.computeData()
    g.es('buffers...')
    for name in self.nameList:
        g.es('',name)

def listBuffersAlphabetically (self,event):

    '''List all buffers (node headlines), in alphabetical order.
    Nodes with the same headline are disambiguated by giving their parent or child index.'''

    self.computeData()
    names = self.nameList[:] ; names.sort()

    g.es('buffers...')
    for name in names:
        g.es('',name)
#@+node:ekr.20050920084036.39: *6* prependToBuffer
def prependToBuffer (self,event):

    '''Add the selected body text to the start of the body text of a named buffer (node).'''

    w = self.editWidget(event) # Sets self.w
    if w:
        self.k.setLabelBlue('Prepend to buffer: ')
        self.getBufferName(event,self.prependToBufferFinisher)

def prependToBufferFinisher (self,name):

    c,w = self.c,self.w
    s = w.getSelectedText()
    p = self.findBuffer(name)
    if s and p:
        c.selectPosition(p)
        self.beginCommand('prepend-to-buffer: %s' % p.h)
        w.insert(0,s)
        w.setInsertPoint(0)
        w.seeInsertPoint()
        self.endCommand()
        c.redraw_after_icons_changed()
        c.recolor()
#@+node:ekr.20050920084036.43: *6* renameBuffer
def renameBuffer (self,event):

    '''Rename a buffer, i.e., change a node's headline.'''

    g.es('rename-buffer not ready yet')
    if 0:
        self.k.setLabelBlue('Rename buffer from: ')
        self.getBufferName(event,self.renameBufferFinisher1)

def renameBufferFinisher1 (self,name):

    self.fromName = name
    self.k.setLabelBlue('Rename buffer from: %s to: ' % (name))
    event = None
    self.getBufferName(event,self.renameBufferFinisher2)

def renameBufferFinisher2 (self,name):

    c = self.c ; p = self.findBuffer(self.fromName)
    if p:
        c.endEditing()
        c.setHeadString(p,name)
        c.redraw(p)
#@+node:ekr.20050920084036.40: *6* switchToBuffer
def switchToBuffer (self,event):

    '''Select a buffer (node) by its name (headline).'''

    self.k.setLabelBlue('Switch to buffer: ')
    self.getBufferName(event,self.switchToBufferFinisher)

def switchToBufferFinisher (self,name):

    c = self.c ; p = self.findBuffer(name)
    if p:
        c.selectPosition(p)
        c.redraw_after_select(p)

#@+node:ekr.20050927102133.1: *5* Utils
#@+node:ekr.20051215121416: *6* computeData
def computeData (self):

    self.nameList = []
    self.names = {} ; self.tnodes = {}

    for p in self.c.all_unique_positions():
        h = p.h.strip()
        v = p.v
        nameList = self.names.get(h,[])
        if nameList:
            if p.parent():
                key = '%s, parent: %s' % (h,p.parent().h)
            else:
                key = '%s, child index: %d' % (h,p.childIndex())
        else:
            key = h
        self.nameList.append(key)
        self.tnodes[key] = v
        nameList.append(key)
        self.names[h] = nameList
#@+node:ekr.20051215164823: *6* findBuffer
def findBuffer (self,name):

    v = self.tnodes.get(name)

    for p in self.c.all_unique_positions():
        if p.v == v:
            return p

    g.trace("Can't happen",name)
    return None
#@+node:ekr.20050927093851: *6* getBufferName (added event arg)
def getBufferName (self,event,finisher):

    '''Get a buffer name into k.arg and call k.setState(kind,n,handler).'''

    c,k = self.c,self.k
    state = k.getState('getBufferName')

    if state == 0:
        self.computeData()
        self.getBufferNameFinisher = finisher
        prefix = k.getLabel()
        k.getArg(event,'getBufferName',1,self.getBufferName,
            prefix=prefix,tabList=self.nameList)
    else:
        k.resetLabel()
        k.clearState()
        finisher = self.getBufferNameFinisher
        self.getBufferNameFinisher = None
        finisher(k.arg)
#@+node:ekr.20070522085324: *4* ChapterCommandsClass
class ChapterCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20070522085340: *5*  ctor (ChapterCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    # c.chapterController does not exist yet.
#@+node:ekr.20070522085429: *5*  getPublicCommands (ChapterCommandsClass)
def getPublicCommands (self):

    c = self.c ; cc = c.chapterController

    if cc:
        return {
            'chapter-clone-node-to':    cc.cloneNodeToChapter,
            'chapter-convert-node-to':  cc.convertNodeToChapter,
            'chapter-copy-node-to':     cc.copyNodeToChapter,
            'chapter-create':           cc.createChapter,
            'chapter-create-from-node': cc.createChapterFromNode,
            'chapter-move-node-to':     cc.moveNodeToChapter,
            'chapter-remove':           cc.removeChapter,
            'chapter-rename':           cc.renameChapter,
            'chapter-select':           cc.selectChapter,
        }
    else:
        return {}
#@+node:ekr.20050920084036.150: *4* ControlCommandsClass
class ControlCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20050920084036.151: *5*  ctor (ControlCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.payload = None
#@+node:ekr.20050920084036.152: *5*  getPublicCommands
def getPublicCommands (self):

    k = self.c.k

    return {

        # Miscellaneous.
        'advertised-undo':          self.advertizedUndo,
        'iconify-frame':            self.iconifyFrame, # Same as suspend.
        'keyboard-quit':            k and k.keyboardQuit,
        'save-buffers-kill-leo':    self.saveBuffersKillLeo,
        'set-silent-mode':          self.setSilentMode,
        'suspend':                  self.suspend,
        'act-on-node':              self.actOnNode,

        # Plugin info.
        'print-plugin-handlers':    self.printPluginHandlers,
        'print-plugins-info':       self.printPluginsInfo,

        # Shell commands.
        'shell-command':            self.shellCommand,
        'shell-command-on-region':  self.shellCommandOnRegion,
    }
#@+node:ekr.20050922110030: *5* advertizedUndo
def advertizedUndo (self,event):

    '''Undo the previous command.'''

    self.c.undoer.undo()
#@+node:ekr.20050920084036.160: *5* executeSubprocess
def executeSubprocess (self,event,command):
    '''Execute a command in a separate process.'''
    k = self.k
    try:
        args = shlex.split(g.toEncodedString(command))
        subprocess.Popen(args).wait()
    except Exception:
        g.es_exception()
    k.keyboardQuit()
        # Inits vim mode too.
    g.es('Done: %s' % command)
#@+node:ekr.20070429090859: *5* print plugins info...
def printPluginHandlers (self,event=None):

    '''Print the handlers for each plugin.'''

    g.app.pluginsController.printHandlers(self.c)

def printPlugins (self,event=None):

    '''Print the file name responsible for loading a plugin.

    This is the first .leo file containing an @enabled-plugins node
    that enables the plugin.'''

    g.app.pluginsController.printPlugins(self.c)

def printPluginsInfo (self,event=None):

    '''Print the file name responsible for loading a plugin.

    This is the first .leo file containing an @enabled-plugins node
    that enables the plugin.'''

    g.app.pluginsController.printPluginsInfo(self.c)
#@+node:ekr.20060603161041: *5* setSilentMode
def setSilentMode (self,event=None):

    '''Set the mode to be run silently, without the minibuffer.
    The only use for this command is to put the following in an @mode node::

        --> set-silent-mode'''

    self.c.k.silentMode = True
#@+node:ekr.20050920084036.158: *5* shellCommand
def shellCommand (self,event):
    '''Execute a shell command.'''
    k = self.k
    state = k.getState('shell-command')
    if state == 0:
        k.setLabelBlue('shell-command: ')
        k.getArg(event,'shell-command',1,self.shellCommand)
    else:
        command = k.arg
        # k.commandName = 'shell-command: %s' % command
        # k.clearState()
        self.executeSubprocess(event,command)
#@+node:ekr.20050930112126: *5* shellCommandOnRegion
def shellCommandOnRegion (self,event):
    '''Execute a command taken from the selected text in a separate process.'''
    k = self.k
    w = self.editWidget(event)
    if w:
        if w.hasSelection():
            command = w.getSelectedText()
            # k.commandName = 'shell-command: %s' % command
            self.executeSubprocess(event,command)
        else:
            # k.clearState()
            g.es('No text selected')
    k.keyboardQuit()
#@+node:ville.20090222184600.2: *5* actOnNode
def actOnNode(self, event):
    """ Execute node-specific action (typically defined by plugins)

    Actual behaviour is to be defined by plugins.

    Here's how to define actions for nodes in your plugins::

        import leo.core.leoPlugins
        def act_print_upcase(c,p,event):
            if not p.h.startswith('@up'):
                raise leo.core.leoPlugins.TryNext
            p.h = p.h.upper()

        g.act_on_node.add(act_print_upcase)        

    This will upcase the headline when it starts with @up.            

    """
    g.act_on_node(self.c,self.c.p,event)
#@+node:ekr.20050920084036.155: *5* shutdown, saveBuffersKillEmacs & setShutdownHook
def shutdown (self,event):

    '''Quit Leo, prompting to save any unsaved files first.'''

    g.app.onQuit()

saveBuffersKillLeo = shutdown
#@+node:ekr.20050920084036.153: *5* suspend & iconifyFrame
def suspend (self,event):

    '''Minimize the present Leo window.'''

    w = self.editWidget(event)
    if not w: return
    self.c.frame.top.iconify()

def iconifyFrame (self,event):

    '''Minimize the present Leo window.'''

    self.suspend(event)
#@+node:ekr.20060127162818.1: *4* DebugCommandsClass
class DebugCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20060127162921: *5*  ctor (DebugCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.
#@+node:ekr.20060127163325: *5*  getPublicCommands
def getPublicCommands (self):

    return {

        # debugging.
        'debug':        self.invoke_debugger,
        'pdb':          self.pdb,
        'print-focus':  self.printFocus,

        # Tracing of garbase collecor.
        'gc-collect-garbage':       self.collectGarbage,
        'gc-dump-all-objects':      self.dumpAllObjects,
        'gc-dump-new-objects':      self.dumpNewObjects,
        'gc-dump-objects-verbose':  self.verboseDumpObjects,
        'gc-print-summary':         self.printGcSummary,
        'gc-trace-disable':         self.disableGcTrace,
        'gc-trace-enable':          self.enableGcTrace,

        # Unit tests run externally: deprecated.
        'run-all-unit-tests-externally':        self.runAllUnitTestsExternally,
            # was 'run-all-unit-tests.
        'run-marked-unit-tests-externally':     self.runMarkedUnitTestsExternally,
            # 2011/10/31: new.
        'run-selected-unit-tests-externally':   self.runSelectedUnitTestsExternally,
            # was 'run-unit-tests.

        # Unit tests run locally.
        'run-all-unit-tests-locally':       self.runAllUnitTestsLocally,
        'run-marked-unit-tests-locally':    self.runMarkedUnitTestsLocally,
            # 2011/10/31: new.
        'run-selected-unit-tests-locally':  self.runSelectedUnitTestsLocally,
    }
#@+node:ekr.20060205050659: *5* collectGarbage
def collectGarbage (self,event=None):

    """Run Python's Gargabe Collector."""

    g.collectGarbage()
#@+node:ekr.20060519003651: *5* invoke_debugger & helper
def invoke_debugger (self,event=None):

    '''Start an external debugger in another process to debug a script.
    The script is the presently selected text or then entire tree's script.'''

    c = self.c ; p = c.p
    python = sys.executable
    script = g.getScript(c,p)
    winpdb = self.findDebugger()
    if not winpdb: return

    #check for doctest examples
    try:
        import doctest
        parser = doctest.DocTestParser()
        examples = parser.get_examples(script)

        # if this is doctest, extract the examples as a script
        if len(examples) > 0:
            script = doctest.script_from_examples(script)
    except ImportError:
        pass

    # special case; debug code may include g.es("info string").
    # insert code fragment to make this expression legal outside Leo.
    hide_ges = "class G:\n def es(s,c=None):\n  pass\ng = G()\n"
    script = hide_ges + script

    # Create a temp file from the presently selected node.
    filename = c.writeScriptFile(script)
    if not filename: return

    # Invoke the debugger, retaining the present environment.
    os.chdir(g.app.loadDir)
    if False and subprocess:
        cmdline = '%s %s -t %s' % (python,winpdb,filename)
        subprocess.Popen(cmdline)
    else:
        args = [sys.executable, winpdb, '-t', filename]
        os.spawnv(os.P_NOWAIT, python, args)
#@+node:ekr.20060521140213: *6* findDebugger
def findDebugger (self):

    '''Find the debugger using settings.'''

    c = self.c
    pythonDir = g.os_path_dirname(sys.executable)

    debuggers = (
        c.config.getString('debugger_path'),
        g.os_path_join(pythonDir,'Lib','site-packages','winpdb.py'), # winpdb 1.1.2 or newer
        g.os_path_join(pythonDir,'scripts','_winpdb.py'), # oder version.
    )

    for debugger in debuggers:
        if debugger:
            debugger = c.os_path_finalize(debugger)
            if g.os_path_exists(debugger):
                return debugger
            else:
                g.warning('debugger does not exist:',debugger)
    g.es('no debugger found.')
    return None
#@+node:ekr.20060202160523: *5* dumpAll/New/VerboseObjects
def dumpAllObjects (self,event=None):

    '''Print a summary of all existing Python objects.'''

    old = g.trace_gc
    g.trace_gc = True
    g.printGcAll()
    g.trace_gc = old

def dumpNewObjects (self,event=None):

    '''Print a summary of all Python objects created
    since the last time Python's Garbage collector was run.'''

    old = g.trace_gc
    g.trace_gc = True
    g.printGcObjects()
    g.trace_gc = old

def verboseDumpObjects (self,event=None):

    '''Print a more verbose listing of all existing Python objects.'''

    old = g.trace_gc
    g.trace_gc = True
    g.printGcVerbose()
    g.trace_gc = old
#@+node:ekr.20060127163325.1: *5* enable/disableGcTrace
def disableGcTrace (self,event=None):

    '''Enable tracing of Python's Garbage Collector.'''

    g.trace_gc = False


def enableGcTrace (self,event=None):

    '''Disable tracing of Python's Garbage Collector.'''

    g.trace_gc = True
    g.enable_gc_debug()

    if g.trace_gc_verbose:
        g.blue('enabled verbose gc stats')
    else:
        g.blue('enabled brief gc stats')
#@+node:ekr.20060202154734: *5* freeTreeWidgets
def freeTreeWidgets (self,event=None):

    '''Free all widgets used in Leo's outline pane.'''

    c = self.c

    c.frame.tree.destroyWidgets()
    c.redraw()
#@+node:ekr.20090226080753.8: *5* pdb
def pdb (self,event=None):

    '''Fall into pdb.'''

    g.pdb()
#@+node:ekr.20060210100432: *5* printFocus
# Doesn't work if the focus isn't in a pane with bindings!

def printFocus (self,event=None):

    '''Print information about the requested focus (for debugging).'''

    c = self.c

    g.es_print('      hasFocusWidget:',c.widget_name(c.hasFocusWidget))
    g.es_print('requestedFocusWidget:',c.widget_name(c.requestedFocusWidget))
    g.es_print('           get_focus:',c.widget_name(c.get_focus()))
#@+node:ekr.20060205043324.3: *5* printGcSummary
def printGcSummary (self,event=None):

    '''Print a brief summary of all Python objects.'''

    g.printGcSummary()
#@+node:ekr.20060202133313: *5* printStats
def printStats (self,event=None):

    '''Print statistics about the objects that Leo is using.'''

    c = self.c
    c.frame.tree.showStats()
    self.dumpAllObjects()
#@+node:ekr.20060328121145: *5* runUnitTest commands
def runAllUnitTestsLocally (self,event=None):
    '''Run all unit tests contained in the presently selected outline.
    Tests are run in the outline's process, so tests *can* change the outline.'''
    self.c.testManager.doTests(all=True)

def runMarkedUnitTestsLocally (self,event=None):
    '''Run marked unit tests in the outline.
    Tests are run in the outline's process, so tests *can* change the outline.'''
    self.c.testManager.doTests(all=True,marked=True)

def runSelectedUnitTestsLocally (self,event=None):
    '''Run all unit tests contained in the presently selected outline.
    Tests are run in the outline's process, so tests *can* change the outline.'''
    self.c.testManager.doTests(all=False,marked=False)

# Externally run tests...

def runAllUnitTestsExternally (self,event=None):
    '''Run all unit tests contained in the entire outline.
    Tests are run in an external process, so tests *cannot* change the outline.'''
    self.c.testManager.runTestsExternally(all=True,marked=False)

def runMarkedUnitTestsExternally(self,event=None):
    '''Run all marked unit tests in the outline.
    Tests are run in an external process, so tests *cannot* change the outline.'''
    self.c.testManager.runTestsExternally(all=True,marked=True)

def runSelectedUnitTestsExternally(self,event=None):
    '''Run all unit tests contained in the presently selected outline
    Tests are run in an external process, so tests *cannot* change the outline.'''
    self.c.testManager.runTestsExternally(all=False,marked=False)
#@+node:ekr.20050920084036.53: *4* EditCommandsClass
class EditCommandsClass (BaseEditCommandsClass):

    '''Contains editing commands with little or no state.'''

    @others
#@+node:ekr.20050929155208: *5*  birth
#@+node:ekr.20050920084036.54: *6*  ctor (EditCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.ccolumn = '0'   # For comment column functions.
    self.extendMode = False # True: all cursor move commands extend the selection.
    self.fillPrefix = '' # For fill prefix functions.
    self.fillColumn = 0 # For line centering.
        # Set by the set-fill-column command.
        # If zero, @pagewidth value is used.
    self.moveSpotNode = None # A VNode.
    self.moveSpot = None # For retaining preferred column when moving up or down.
    self.moveCol = None # For retaining preferred column when moving up or down.
    self.sampleWidget = None # Created later.
    self.swapSpots = []
    self._useRegex = False # For replace-string
    self.w = None # For use by state handlers.

    # Settings...
    cf = c.config
    self.autocompleteBrackets   = cf.getBool('autocomplete-brackets')
    self.bracketsFlashBg        = cf.getColor('flash-brackets-background-color')
    self.bracketsFlashCount     = cf.getInt('flash-brackets-count')
    self.bracketsFlashDelay     = cf.getInt('flash-brackets-delay')
    self.bracketsFlashFg        = cf.getColor('flash-brackets-foreground-color')
    self.flashMatchingBrackets  = cf.getBool('flash-matching-brackets')
    self.smartAutoIndent        = cf.getBool('smart_auto_indent')
    self.openBracketsList       = cf.getString('open_flash_brackets')  or '([{'
    self.closeBracketsList      = cf.getString('close_flash_brackets') or ')]}'

    self.initBracketMatcher(c)
#@+node:ekr.20050920084036.55: *6*  getPublicCommands (EditCommandsClass)
def getPublicCommands (self):        

    c = self.c

    return {
        'activate-cmds-menu':                   self.activateCmdsMenu,
        'activate-edit-menu':                   self.activateEditMenu,
        'activate-file-menu':                   self.activateFileMenu,
        'activate-help-menu':                   self.activateHelpMenu,
        'activate-outline-menu':                self.activateOutlineMenu,
        'activate-plugins-menu':                self.activatePluginsMenu,
        'activate-window-menu':                 self.activateWindowMenu,
        'add-editor':                           c.frame.body and c.frame.body.addEditor,
        'add-space-to-lines':                   self.addSpaceToLines,
        'add-tab-to-lines':                     self.addTabToLines, 
        'back-to-indentation':                  self.backToIndentation,
        'back-to-home':                         self.backToHome,
        'back-to-home-extend-selection':        self.backToHomeExtendSelection,
        'back-char':                            self.backCharacter,
        'back-char-extend-selection':           self.backCharacterExtendSelection,
        'back-page':                            self.backPage,
        'back-page-extend-selection':           self.backPageExtendSelection,
        'back-paragraph':                       self.backwardParagraph,
        'back-paragraph-extend-selection':      self.backwardParagraphExtendSelection,
        'back-sentence':                        self.backSentence,
        'back-sentence-extend-selection':       self.backSentenceExtendSelection,
        'back-word':                            self.backwardWord,
        'back-word-extend-selection':           self.backwardWordExtendSelection,
        'back-word-smart':                      self.backwardWordSmart,
        'back-word-smart-extend-selection':     self.backwardWordSmartExtendSelection,
        'backward-delete-char':                 self.backwardDeleteCharacter,
        'backward-delete-word':                 self.backwardDeleteWord,
        'backward-delete-word-smart':           self.backwardDeleteWordSmart,
        'backward-kill-paragraph':              self.backwardKillParagraph,
        'backward-find-character':              self.backwardFindCharacter,
        'backward-find-character-extend-selection': self.backwardFindCharacterExtendSelection,
        'beginning-of-buffer':                  self.beginningOfBuffer,
        'beginning-of-buffer-extend-selection': self.beginningOfBufferExtendSelection,
        'beginning-of-line':                    self.beginningOfLine,
        'beginning-of-line-extend-selection':   self.beginningOfLineExtendSelection,
        'c-to-python':                          self.cToPy,
        'capitalize-word':                      self.capitalizeWord,
        'center-line':                          self.centerLine,
        'center-region':                        self.centerRegion,
        'clean-all-lines':                      self.cleanAllLines,
        'clean-lines':                          self.cleanLines,
        'clear-all-caches':                     self.clearAllCaches,
        'clear-all-uas':                        self.clearAllUas,
        'clear-cache':                          self.clearCache,
        'clear-extend-mode':                    self.clearExtendMode,
        'clear-node-uas':                       self.clearNodeUas,
        'clear-selected-text':                  self.clearSelectedText,
        'click-click-box':                      self.clickClickBox,
        'click-headline':                       self.clickHeadline,
        'click-icon-box':                       self.clickIconBox,
        'clone-marked-nodes':                   c.cloneMarked,
        'cls':                                  g.cls,
        'contract-body-pane':                   c.frame.contractBodyPane,
        'contract-log-pane':                    c.frame.contractLogPane,
        'contract-outline-pane':                c.frame.contractOutlinePane,
        'contract-pane':                        c.frame.contractPane,
        'count-region':                         self.countRegion,
        'ctrl-click-icon':                      self.ctrlClickIconBox,
        'cycle-focus':                          self.cycleFocus,
        # 'cycle-all-focus':                    self.cycleAllFocus,
            # Replaced by focus-to-x commands.
        'cycle-editor-focus':                   c.frame.body.cycleEditorFocus,
        'cycle-log-focus':                      c.frame.log.cycleTabFocus,
        # 'delete-all-icons':                   self.deleteAllIcons,
        'delete-char':                          self.deleteNextChar,
        'delete-editor':                        c.frame.body.deleteEditor,
        'delete-first-icon':                    self.deleteFirstIcon,
        'delete-indentation':                   self.deleteIndentation,
        'delete-last-icon':                     self.deleteLastIcon,
        'delete-marked-nodes':                  c.deleteMarked,
        'delete-node-icons':                    self.deleteNodeIcons,
        'delete-spaces':                        self.deleteSpaces,
        'delete-word':                          self.deleteWord,
        'delete-word-smart':                    self.deleteWordSmart,
        'do-nothing':                           self.doNothing,
        'downcase-region':                      self.downCaseRegion,
        'downcase-word':                        self.downCaseWord,
        'double-click-headline':                self.doubleClickHeadline,
        'double-click-icon-box':                self.doubleClickIconBox,
        'end-of-buffer':                        self.endOfBuffer,
        'end-of-buffer-extend-selection':       self.endOfBufferExtendSelection,
        'end-of-line':                          self.endOfLine,
        'end-of-line-extend-selection':         self.endOfLineExtendSelection,
        'escape':                               self.watchEscape,
        'eval-expression':                      self.evalExpression,
        'exchange-point-mark':                  self.exchangePointMark,
        'expand-body-pane':                     c.frame.expandBodyPane,
        'expand-log-pane':                      c.frame.expandLogPane,
        'expand-outline-pane':                  c.frame.expandOutlinePane,
        'expand-pane':                          c.frame.expandPane,
        'extend-to-line':                       self.extendToLine,
        'extend-to-paragraph':                  self.extendToParagraph,
        'extend-to-sentence':                   self.extendToSentence,
        'extend-to-word':                       self.extendToWord,
        'fill-paragraph':                       self.fillParagraph,
        'fill-region':                          self.fillRegion,
        'fill-region-as-paragraph':             self.fillRegionAsParagraph,
        'find-character':                       self.findCharacter,
        'find-character-extend-selection':      self.findCharacterExtendSelection,
        'find-word':                            self.findWord,
        'find-word-in-line':                    self.findWordInLine,
        'flush-lines':                          self.flushLines,
        'focus-to-body':                        self.focusToBody,
        'focus-to-log':                         self.focusToLog,
        'focus-to-minibuffer':                  self.focusToMinibuffer,
        'focus-to-tree':                        self.focusToTree,
        'forward-char':                         self.forwardCharacter,
        'forward-char-extend-selection':        self.forwardCharacterExtendSelection,
        'forward-page':                         self.forwardPage,
        'forward-page-extend-selection':        self.forwardPageExtendSelection,
        'forward-paragraph':                    self.forwardParagraph,
        'forward-paragraph-extend-selection':   self.forwardParagraphExtendSelection,
        'forward-sentence':                     self.forwardSentence,
        'forward-sentence-extend-selection':    self.forwardSentenceExtendSelection,
        'forward-end-word':                     self.forwardEndWord, # New in Leo 4.4.2.
        'forward-end-word-extend-selection':    self.forwardEndWordExtendSelection, # New in Leo 4.4.2.
        'forward-word':                         self.forwardWord,
        'forward-word-extend-selection':        self.forwardWordExtendSelection,
        'forward-word-smart':                   self.forwardWordSmart,
        'forward-word-smart-extend-selection':  self.forwardWordSmartExtendSelection,
        'fully-expand-body-pane':               c.frame.fullyExpandBodyPane,
        'fully-expand-log-pane':                c.frame.fullyExpandLogPane,
        'fully-expand-pane':                    c.frame.fullyExpandPane,
        'fully-expand-outline-pane':            c.frame.fullyExpandOutlinePane,
        'goto-char':                            self.gotoCharacter,
        'goto-global-line':                     self.gotoGlobalLine,
        'goto-line':                            self.gotoLine,
        'hide-body-pane':                       c.frame.hideBodyPane,
        'hide-log-pane':                        c.frame.hideLogPane,
        'hide-pane':                            c.frame.hidePane,
        'hide-outline-pane':                    c.frame.hideOutlinePane,
        'how-many':                             self.howMany,
        # Use indentBody in leoCommands.py
        'indent-relative':                      self.indentRelative,
        'indent-rigidly':                       self.tabIndentRegion,
        'indent-to-comment-column':             self.indentToCommentColumn,
        'insert-hard-tab':                      self.insertHardTab,
        'insert-icon':                          self.insertIcon,
        'insert-file-name':                     self.insertFileName,
        'insert-headline-time':                 self.insertHeadlineTime,
        'insert-newline':                       self.insertNewline,
        'insert-parentheses':                   self.insertParentheses,
        'insert-soft-tab':                      self.insertSoftTab,
        'keep-lines':                           self.keepLines,
        'kill-paragraph':                       self.killParagraph,
        'line-number':                          self.lineNumber,
        'move-lines-down':                      self.moveLinesDown,
        'move-lines-up':                        self.moveLinesUp,
        'move-marked-nodes':                    c.moveMarked,
        'move-past-close':                      self.movePastClose,
        'move-past-close-extend-selection':     self.movePastCloseExtendSelection,
        'newline-and-indent':                   self.insertNewLineAndTab,
        'next-line':                            self.nextLine,
        'next-line-extend-selection':           self.nextLineExtendSelection,
        'previous-line':                        self.prevLine,
        'previous-line-extend-selection':       self.prevLineExtendSelection,
        'print-all-uas':                        self.printAllUas,
        'print-node-uas':                       self.printUas,
        'remove-blank-lines':                   self.removeBlankLines,
        'remove-space-from-lines':              self.removeSpaceFromLines,
        'remove-tab-from-lines':                self.removeTabFromLines,
        'replace-current-character':            self.replaceCurrentCharacter,
        'reverse-region':                       self.reverseRegion,
        'reverse-sort-lines':                   self.reverseSortLines,
        'reverse-sort-lines-ignoring-case':     self.reverseSortLinesIgnoringCase,
        'scroll-down-half-page':                self.scrollDownHalfPage,                
        'scroll-down-line':                     self.scrollDownLine,
        'scroll-down-page':                     self.scrollDownPage,
        'scroll-outline-down-line':             self.scrollOutlineDownLine,
        'scroll-outline-down-page':             self.scrollOutlineDownPage,
        'scroll-outline-left':                  self.scrollOutlineLeft,
        'scroll-outline-right':                 self.scrollOutlineRight,
        'scroll-outline-up-line':               self.scrollOutlineUpLine,
        'scroll-outline-up-page':               self.scrollOutlineUpPage,
        'scroll-up-half-page':                  self.scrollUpHalfPage,                        
        'scroll-up-line':                       self.scrollUpLine,
        'scroll-up-page':                       self.scrollUpPage,
        'select-all':                           self.selectAllText,
        'select-to-matching-bracket':           self.selectToMatchingBracket,
        # Exists, but can not be executed via the minibuffer.
        'self-insert-command':                  self.selfInsertCommand,
        'set-comment-column':                   self.setCommentColumn,
        'set-extend-mode':                      self.setExtendMode,
        'set-fill-column':                      self.setFillColumn,
        'set-fill-prefix':                      self.setFillPrefix,
        #'set-mark-command':                    self.setRegion,
        'set-ua':                               self.setUa,
        'show-colors':                          self.showColors,
        'show-fonts':                           self.showFonts,
        'simulate-begin-drag':                  self.simulateBeginDrag,
        'simulate-end-drag':                    self.simulateEndDrag,
        'sort-columns':                         self.sortColumns,
        'sort-fields':                          self.sortFields,
        'sort-lines':                           self.sortLines,
        'sort-lines-ignoring-case':             self.sortLinesIgnoringCase,
        'split-line':                           self.splitLine,
        'tabify':                               self.tabify,
        'toggle-extend-mode':                   self.toggleExtendMode,
        'toggle-case-region':                   self.toggleCaseRegion,
        'transpose-chars':                      self.transposeCharacters,
        'transpose-lines':                      self.transposeLines,
        'transpose-words':                      self.transposeWords,
        'typescript-to-py':                     self.tsToPy,
        'untabify':                             self.untabify,
        'upcase-region':                        self.upCaseRegion,
        'upcase-word':                          self.upCaseWord,
        'view-lossage':                         self.viewLossage,
        'what-line':                            self.whatLine,
    }
#@+node:ekr.20061012113455: *6* doNothing
def doNothing (self,event):

    '''A placeholder command, useful for testing bindings.'''

    pass
#@+node:ekr.20110916215321.6709: *5* brackets (leoEditCommands)
#@+node:ekr.20110916215321.6708: *6* selectToMatchingBracket (leoEditCommands)
def selectToMatchingBracket (self,event):

    '''Select text that matches the bracket near the cursor.'''

    c = self.c
    w = self.editWidget(event)
    if not w: return
    i = w.getInsertPoint()
    s = w.getAllText()
    allBrackets = self.openBracketsList + self.closeBracketsList
    if i < len(s) and s[i] in allBrackets:
        ch = s[i]
    elif i > 0 and s[i-1] in allBrackets:
        i -= 1
        ch = s[i]
    else:
        g.es('no bracket selected')
        return

    d = {}
    if ch in self.openBracketsList:
        for z in range(len(self.openBracketsList)):
            d [self.openBracketsList[z]] = self.closeBracketsList[z]
        reverse = False # Search forward
    else:
        for z in range(len(self.openBracketsList)):
            d [self.closeBracketsList[z]] = self.openBracketsList[z]
        reverse = True # Search backward
    delim2 = d.get(ch)

    # This should be generalized...
    language = g.findLanguageDirectives(c,c.p)
    if language in ('c','cpp','csharp'):
        j = g.skip_matching_c_delims(s,i,ch,delim2,reverse=reverse)
    else:
        j = g.skip_matching_python_delims(s,i,ch,delim2,reverse=reverse)
    # g.trace(i,j,ch,delim2,reverse,language)
    if j not in (-1,i):
        if reverse:
            i += 1; j += 1
        w.setSelectionRange(i,j,insert=j)
            # 2011/11/21: Bug fix: was ins=j.
        w.see(j)
#@+node:ekr.20121016093159.10182: *5* c/ts/toPY
<< theory of operation >>
<< class To_Python >>
<< class C_To_Python (To_Python) >>
<< class TS_To_Python (To_Python) >>

def cToPy (self,event):

    ''' The c-to-python command converts c or c++ text to python text.
    The conversion is not perfect, but it eliminates a lot of tedious
    text manipulation.'''

    self.C_To_Python(self.c).go()
    self.c.bodyWantsFocus()

def tsToPy (self,event):

    ''' The typescript-to-python command converts typescript text to python
    text. The conversion is not perfect, but it eliminates a lot of tedious
    text manipulation.'''

    # Compiler stats: 35 files, 1489 nodes, 100 to 120 sec.
    self.TS_To_Python(self.c).go()
    self.c.bodyWantsFocus()
#@+node:ekr.20121016093159.10240: *6* << theory of operation >>
@nocolor
@

1. We use a single list, aList, for all changes to the text. This reduces
   stress on the gc. All replacesments are done **in place** in aList.

2. The following pattern ensures replacements do not happen in strings and comments::

    def someScan(self,aList):
        i = 0
        while i < len(aList):
            if self.is_string_or_comment(aList,i):
                i = skip_string_or_comment(aList,i)
            elif < found what we are looking for >:
                <convert what we are looking for, setting i>
            else: i += 1

#@+node:ekr.20121016093159.10183: *6* << class To_Python >>
class To_Python:

    '''The base class for x-to-python commands.'''

    @others
#@+node:ekr.20121016093159.10241: *7* ctor (To_Python)
def __init__ (self,c):

    self.c = c
    self.p = self.c.p.copy()

    aList = g.get_directives_dict_list(self.p)
    self.tab_width = g.scanAtTabwidthDirectives(aList) or 4
#@+node:ekr.20121016093159.10299: *7* go
def go (self):

    import time
    t1 = time.time()
    c = self.c
    u = c.undoer ; undoType = 'typescript-to-python'
    pp = c.CPrettyPrinter(c)

    u.beforeChangeGroup(c.p,undoType)
    changed, dirtyVnodeList = False,[]
    n_files, n_nodes = 0,0
    special = ('class ','module ','@file ','@@file ')
    files = ('@file ','@@file ')
    for p in self.p.self_and_subtree():
        if p.b:
            n_nodes += 1
            if any([p.h.startswith(z) for z in special]):
                g.es_print(p.h)
                if any([p.h.startswith(z) for z in files]):
                    n_files += 1
            bunch = u.beforeChangeNodeContents(p)

            s = pp.indent(p,giveWarnings=False)
            aList = list(s)
            self.convertCodeList(aList)

            s = ''.join(aList)
            if s != p.b:
                p.b = s
                p.v.setDirty()
                dirtyVnodeList.append(p.v)
                u.afterChangeNodeContents(p,undoType,bunch)
                changed = True

    # Call this only once, at end.
    if changed:
        u.afterChangeGroup(c.p,undoType,
            reportFlag=False,dirtyVnodeList=dirtyVnodeList)

    t2 = time.time()
    g.es_print('done! %s files, %s nodes, %2.2f sec' % (n_files,n_nodes,t2-t1))
#@+node:ekr.20121016093159.10245: *7* convertCodeList (must be defined in subclasses)
def convertCodeList(self,aList):

    '''The main search/replace method.'''

    g.trace('must be defined in subclasses.')
#@+node:ekr.20121016093159.10259: *7* Utils
#@+node:ekr.20121016093159.10260: *8* match...
#@+node:ekr.20121016093159.10261: *9* match
def match (self,s,i,pat):

    '''Return True if s[i:] matches the pat string.

    We can't use g.match because s is usually a list.
    '''

    assert pat

    j = 0
    while i+j < len(s) and j < len(pat):
        if s[i+j] == pat[j]:
            j += 1
            if j == len(pat):
                return True
        else:
            return False

    return False
#@+node:ekr.20121016093159.10293: *9* match_word
def match_word (self,s,i,pat):
    '''
    Return True if s[i:] word matches the pat string.

    We can't use g.match_word because s is usually a list
    and g.match_word uses s.find.
    '''
    if self.match(s,i,pat):
        j = i + len(pat)
        if j >= len(s):
            return True
        elif not pat[-1].isalnum():
            # Bug fix 10/16/2012: The pattern terminates the word.
            return True
        else:
            ch = s[j]
            return not ch.isalnum() and ch != '_'
    else:
        return False
#@+node:ekr.20121016093159.10247: *8* insert_not
# This may be defined in subclasses, but is not at present.

def insert_not (self,aList):

    '''Change "!" to "not" except before "="'''

    i = 0
    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif aList[i] == '!' and not self.match(aList,i+1,'='):
            aList[i:i+1] = list('not ')
            i += 4
        else:
            i += 1
#@+node:ekr.20121016093159.10263: *8* is...
#@+node:ekr.20121126103128.10144: *9* is_section_def/ref
def is_section_def (self,p):

    return self.is_section_ref(p.h)

def is_section_ref (self,s):

    n1 = s.find("<<",0)
    n2 = s.find(">>",0)
    return -1 < n1 < n2 and s[n1+2:n2].strip()
#@+node:ekr.20121016093159.10265: *9* is_string_or_comment
def is_string_or_comment (self,s,i):

    # Does range checking.
    m = self.match
    return m(s,i,"'") or m(s,i,'"') or m(s,i,"//") or m(s,i,"/*")
#@+node:ekr.20121016093159.10266: *9* is_ws and is_ws_or_nl
def is_ws (self,ch):
    return ch in ' \t'

def is_ws_or_nl (self,ch):
    return ch in ' \t\n'
#@+node:ekr.20121016093159.10267: *8* prevNonWsChar and prevNonWsOrNlChar
def prevNonWsChar (self,s,i):

    i -= 1
    while i >= 0 and self.is_ws(s[i]):
        i -= 1
    return i

def prevNonWsOrNlChar (self,s,i):

    i -= 1
    while i >= 0 and self.is_ws_or_nl(s[i]):
        i -= 1
    return i
#@+node:ekr.20121016093159.10268: *8* remove...
#@+node:ekr.20121016093159.10269: *9* removeBlankLines
def removeBlankLines (self,aList):

    i = 0
    while i < len(aList):
        j = i
        while j < len(aList) and aList[j] in " \t":
            j += 1
        if j == len(aList) or aList[j] == '\n':
            del aList[i:j+1]
        else:
            i = self.skip_past_line(aList,i)
#@+node:ekr.20121016093159.10270: *9* removeExcessWs
def removeExcessWs (self,aList):

    i = 0
    i = self.removeExcessWsFromLine(aList,i)
    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif self.match(aList,i,'\n'):
            i += 1
            i = self.removeExcessWsFromLine(aList,i)
        else: i += 1
#@+node:ekr.20121016093159.10271: *9* removeExessWsFromLine
def removeExcessWsFromLine (self,aList,i):

    assert(i==0 or aList[i-1] == '\n')
    i = self.skip_ws(aList,i)
        # Retain the leading whitespace.

    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            break # safe
        elif self.match(aList,i,'\n'):
            break
        elif self.match(aList,i,' ') or self.match(aList,i,'\t'):
            # Replace all whitespace by one blank.
            j = self.skip_ws(aList,i)
            assert(j > i)
            aList[i:j] = [' ']
            i += 1 # make sure we don't go past a newline!
        else: i += 1
    return i
#@+node:ekr.20121016093159.10272: *9* removeMatchingBrackets
def removeMatchingBrackets (self,aList, i):

    j = self.skip_to_matching_bracket(aList, i)
    if j > i and j < len(aList):
        # print "del brackets:", ''.join(aList[i:j+1])
        c = aList[j]
        if c == ')' or c == ']' or c == '}':
            del aList[j:j+1]
            del aList[i:i+1]
            # print "returning:", ''.join(aList[i:j])
            return j - 1
        else: return j + 1
    else: return j
#@+node:ekr.20121016093159.10273: *9* removeSemicolonsAtEndOfLines
def removeSemicolonsAtEndOfLines (self,aList):

    i = 0
    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif aList[i] == ';':
            j = self.skip_ws(aList,i+1)
            if (
                j >= len(aList) or
                self.match(aList,j,'\n') or
                self.match(aList,j,'#') or
                self.match(aList,j,"//")
            ):
                del aList[i]
            else: i += 1
        else: i += 1
#@+node:ekr.20121016093159.10274: *9* removeTrailingWs
def removeTrailingWs (self,aList):

    i = 0
    while i < len(aList):
        if self.is_ws(aList[i]):
            j = i
            i = self.skip_ws(aList,i)
            assert(j < i)
            if i >= len(aList) or aList[i] == '\n':
                # print "removing trailing ws:", `i-j`
                del aList[j:i]
                i = j
        else: i += 1
#@+node:ekr.20121016093159.10275: *8* replace... & safe_replace
#@+node:ekr.20121016093159.10276: *9* replace
def replace (self,aList,findString,changeString):

    '''# Replaces all occurances of findString by changeString.
    changeString may be the empty string, but not None.
    '''

    if not findString: return

    changeList = list(changeString)
    i = 0
    while i < len(aList):
        if self.match(aList,i,findString):
            aList[i:i+len(findString)] = changeList
            i += len(changeList)
        else:
            i += 1
#@+node:ekr.20121016093159.10277: *9* replaceComments
def replaceComments (self,aList):

    i = 0
    while i < len(aList):
        # Loop invariant: j > progress at end.
        progress = i
        if self.match(aList,i,"//"):
            aList[i:i+2] = ['#']
            j = self.skip_past_line(aList,i)
        elif self.match(aList,i,"/*"):
            j = self.skip_c_block_comment(aList,i)
            k = i
            while k-1 >= 0 and aList[k-1] in ' \t':
                k -= 1
            assert k == 0 or aList[k-1] not in ' \t'
            lws = ''.join(aList[k:i])
            comment_body = ''.join(aList[i+2:j-2])
            comment_lines = g.splitLines(lws + comment_body)
            comment_lines = self.munge_block_comment(comment_lines)
            comment = '\n'.join(comment_lines) # A list of lines.
            comment_list = list(comment) # A list of characters.
            aList[k:j] = comment_list
            j = k + len(comment_list)
            progress = j - 1 # Disable the check below.
        elif self.match(aList,i,'"') or self.match(aList,i,"'"):
            j = self.skip_string(aList,i)
        else:
            j = i+1

        # Defensive programming.
        if j == progress:
            j += 1
        assert j > progress
        i = j
#@+node:ekr.20121016093159.10278: *10* munge_block_comment
def munge_block_comment (self,comment_lines):

    trace = False
    n = len(comment_lines)
    assert n > 0

    s = comment_lines[0]
    junk,w = g.skip_leading_ws_with_indent(s,0,tab_width=4)

    if n == 1:
        return ['%s# %s' % ((' ' * (w-1)),s.strip())]

    junk,w = g.skip_leading_ws_with_indent(s,0,tab_width=4)
    i,result = 0,[]
    for i in range(len(comment_lines)):
        s = comment_lines[i]
        if s.strip():
            result.append('%s# %s' % ((' ' * w),s.strip()))
        elif i == n-1:
            pass # Omit the line entirely.
        else:
            result.append('') # Add a blank line

    if trace:
        g.trace()
        for z in result: print(repr(z))

    return result
#@+node:ekr.20121016093159.10279: *9* replaceSectionDefs
def replaceSectionDefs (self,aList):

    '''Replaces < < x > > = by @c (at the start of lines).'''

    if not aList: return
    i = 0
    j = self.is_section_def(aList[i])
    if j > 0: aList[i:j] = list("@c ")

    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif self.match(aList,i,"\n"):
            i += 1
            j = self.is_section_def(aList[i])
            if j > i: aList[i:j] = list("@c ")
        else: i += 1
#@+node:ekr.20121016093159.10280: *9* safe_replace
def safe_replace (self,aList,findString,changeString):

    '''Replaces occurances of findString by changeString,
    but only outside of C comments and strings.
    changeString may be the empty string, but not None.
    '''

    if not findString: return

    changeList = list(changeString)
    i = 0
    if findString[0].isalpha(): # use self.match_word
        while i < len(aList):
            if self.is_string_or_comment(aList,i):
                i = self.skip_string_or_comment(aList,i)
            elif self.match_word(aList,i,findString):
                aList[i:i+len(findString)] = changeList
                i += len(changeList)
            else:
                i += 1
    else: #use self.match
        while i < len(aList):
            if self.match(aList,i,findString):
                aList[i:i+len(findString)] = changeList
                i += len(changeList)
            else:
                i += 1
#@+node:ekr.20121016093159.10281: *8* skip
#@+node:ekr.20121016093159.10282: *9* skip_c_block_comment
def skip_c_block_comment (self,s,i):

    # if 'replaceComments' in g.callers():
        # g.trace(repr(''.join(s[i:i+20])))

    assert(self.match(s,i,"/*"))
    i += 2

    while i < len(s):
        if self.match(s,i,"*/"):
            return i + 2
        else:
            i += 1

    return i
#@+node:ekr.20121016093159.10298: *9* skip_line
def skip_line (self,s,i):

    while i < len(s) and s[i] != '\n':
        i += 1
    return i
#@+node:ekr.20121016093159.10283: *9* skip_past_line
def skip_past_line (self,s,i):

    while i < len(s) and s[i] != '\n':
        i += 1
    if i < len(s) and s[i] == '\n':
        i += 1
    return i
#@+node:ekr.20121016093159.10284: *9* skip_past_word
def skip_past_word (self,s,i):

    assert(s[i].isalpha() or s[i]=='~')

    # Kludge: this helps recognize dtors.
    if s[i]=='~':
        i += 1

    while i < len(s):
        ch = s[i]
        if ch.isalnum() or ch =='_':
            i += 1
        else:
            break
    return i
#@+node:ekr.20121016093159.10285: *9* skip_string
def skip_string (self,s,i):

    delim = s[i] # handle either single or double-quoted strings
    assert(delim == '"' or delim == "'")
    i += 1

    while i < len(s):
        if s[i] == delim:
            return i + 1
        elif s[i] == '\\':
            i += 2
        else:
            i += 1
    return i
#@+node:ekr.20121016093159.10286: *9* skip_string_or_comment
def skip_string_or_comment (self,s,i):

    if self.match(s,i,"'") or self.match(s,i,'"'):
        j = self.skip_string(s,i)
    elif self.match(s,i,"//"):
        j = self.skip_past_line(s,i)
    elif self.match(s,i,"/*"):
        j = self.skip_c_block_comment(s,i)
    else: assert(0)

    # g.trace(repr(''.join(s[i:j])))
    return j
#@+node:ekr.20121016093159.10287: *9* skip_to_matching_bracket
def skip_to_matching_bracket (self,s,i):

    ch = s[i]
    if   ch == '(': delim = ')'
    elif ch == '{': delim = '}'
    elif ch == '[': delim = ']'
    else: assert(0)

    i += 1
    while i < len(s):
        ch = s[i]
        if self.is_string_or_comment(s,i):
            i = self.skip_string_or_comment(s,i)
        elif ch == delim:
            return i
        elif ch == '(' or ch == '[' or ch == '{':
            i = self.skip_to_matching_bracket(s,i)
            i += 1 # skip the closing bracket.
        else: i += 1
    return i
#@+node:ekr.20121016093159.10288: *9* skip_ws and skip_ws_and_nl
def skip_ws (self,aList,i):

    while i < len(aList):
        c = aList[i]
        if c == ' ' or c == '\t':
            i += 1
        else: break
    return i

def skip_ws_and_nl (self,aList,i):

    while i < len(aList):
        c = aList[i]
        if c == ' ' or c == '\t' or c == '\n':
            i += 1
        else: break
    return i
#@+node:ekr.20121016093159.10184: *6* << class C_To_Python (To_Python) >>
class C_To_Python (To_Python):

    @others
#@+node:ekr.20110916215321.8057: *7* ctor & helpers (C_to_Python)
def __init__ (self,c):

    c.editCommands.To_Python.__init__(self,c)
        # init the base class

    # Internal state...
    self.class_name = ''
        # The class name for the present function.  Used to modify ivars.

    self.ivars = []
        # List of ivars to be converted to self.ivar


    self.get_user_types()
#@+node:ekr.20110916215321.7984: *8* get_user_types
@nocolor
@

Change the following lists so they contain the types and classes used by your
program. c-to-python converts::

    new aType(...)

to::

    aType(...)

Change ivarsDict so it represents the instance variables (ivars) used by your
program's classes. ivarsDict is a dictionary used to translate ivar i of class c
to self.i. It also translates this->i to self.i.

@c
@color

def get_user_types (self):

    c = self.c

    self.class_list = c.config.getData('c-to-python-class-list') or []

    self.type_list  = (
        c.config.getData('c-to-python-type-list') or
        ["char", "void", "short", "long", "int", "double", "float"]
    )
    aList = c.config.getData('c-to-python-ivars-dict')
    if aList:
        self.ivars_dict = self.parse_ivars_data(aList)
    else:
        self.ivars_dict = {}

    if 0:
        #g.trace('class_list',self.class_list)
        #g.trace('type_list',self.type_list)
        g.trace('ivars_dict...')
        d = self.ivars_dict
        keys = list(d.keys())
        for key in sorted(keys):
            print('%s:' % (key))
            for val in d.get(key):
                print('  %s' % (val))

#@+node:ekr.20110917104720.6877: *8* parse_ivars_data
def parse_ivars_data (self,aList):

    d,key = {},None
    aList = [z.strip() for z in aList if z.strip()]
    for s in aList:
        if s.endswith(':'):
            key = s[:-1].strip()
        elif key:
            ivars = [z.strip() for z in s.split(',') if z.strip()]
            aList = d.get(key,[])
            aList.extend(ivars)
            d [key] = aList
        else:
            g.error('invalid @data c-to-python-ivars-dict',repr(s))
            return {}

    return d
#@+node:ekr.20110916215321.7997: *7* convertCodeList (C_To_Python) & helpers
def convertCodeList(self,aList):

    r,sr = self.replace,self.safe_replace

    # First...
    r(aList, "\r", '')
    # self.convertLeadingBlanks(aList) # Now done by indent.
    # if leoFlag: replaceSectionDefs(aList)
    self.mungeAllFunctions(aList)

    # Next...
    if 1:
        # CC2 stuff:
        sr(aList, "TRACEPB",   "if trace: g.trace")
        sr(aList, "TRACEPN",   "if trace: g.trace")
        sr(aList, "TRACEPX",   "if trace: g.trace")
        sr(aList, "TICKB",     "if trace: g.trace")
        sr(aList, "TICKN",     "if trace: g.trace")
        sr(aList, "TICKX",     "if trace: g.trace")
        sr(aList, "g.trace(ftag,", "g.trace(")
        sr(aList, "ASSERT_TRACE", "assert")

    sr(aList, "ASSERT","assert")
    sr(aList, " -> ", '.')
    sr(aList, "->", '.')
    sr(aList, " . ", '.')
    sr(aList, "this.self", "self")
    sr(aList, "{", '')
    sr(aList, "}", '')
    sr(aList, "#if", "if")
    sr(aList, "#else", "else")
    sr(aList, "#endif", '')
    sr(aList, "else if", "elif")
    sr(aList, "else", "else:")
    sr(aList, "&&", " and ")
    sr(aList, "||", " or ")
    sr(aList, "TRUE", "True")
    sr(aList, "FALSE", "False")
    sr(aList, "NULL", "None")
    sr(aList, "this", "self")
    sr(aList, "try", "try:")
    sr(aList, "catch", "except:")
    # if leoFlag: sr(aList, "@code", "@c")

    # Next...
    self.handle_all_keywords(aList)
    self.insert_not(aList)
    self.removeSemicolonsAtEndOfLines(aList)
        # after processing for keywords

    # Last...
    # if firstPart and leoFlag: removeLeadingAtCode(aList)
    self.removeBlankLines(aList)
    self.removeExcessWs(aList)
    # your taste may vary: in Python I don't like extra whitespace
    sr(aList, " :", ":") 
    sr(aList, ", ", ",")
    sr(aList, " ,", ",")
    sr(aList, " (", "(")
    sr(aList, "( ", "(")
    sr(aList, " )", ")")
    sr(aList, ") ", ")")
    sr(aList, "@language c","@language python")
    self.replaceComments(aList) # should follow all calls to safe_replace
    self.removeTrailingWs(aList)
    r(aList, "\t ", "\t") # happens when deleting declarations.
#@+node:ekr.20110916215321.8011: *8* handle_all_keywords
def handle_all_keywords (self,aList):

    '''
    converts if ( x ) to if x:
    converts while ( x ) to while x:
    '''

    i = 0
    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif (
            self.match_word(aList,i,"if") or
            self.match_word(aList,i,"while") or
            self.match_word(aList,i,"for") or
            self.match_word(aList,i,"elif")
        ):
            i = self.handle_keyword(aList,i)
        else:
            i += 1
    # print "handAllKeywords2:", ''.join(aList)
#@+node:ekr.20110916215321.8012: *9* handle_keyword
def handle_keyword (self,aList,i):

    if self.match_word(aList,i,"if"):
        i += 2
    elif self.match_word(aList,i,"elif"):
        i += 4
    elif self.match_word(aList,i,"while"):
        i += 5
    elif self.match_word(aList,i,"for"):
        i += 3
    else: assert(0)

    # Make sure one space follows the keyword.
    k = i
    i = self.skip_ws(aList,i)
    if k == i:
        c = aList[i]
        aList[i:i+1] = [ ' ', c ]
        i += 1

    # Remove '(' and matching ')' and add a ':'
    if aList[i] == "(":
        # Look ahead.  Don't remove if we span a line.
        j = self.skip_to_matching_bracket(aList, i)
        k = i
        found = False
        while k < j and not found:
            found = aList[k] == '\n'
            k += 1
        if not found:
            j = self.removeMatchingBrackets(aList,i)
        if j > i and j < len(aList):
            ch = aList[j]
            aList[j:j+1] = [ch,":", " "]
            j = j + 2
        return j
    return i
#@+node:ekr.20110916215321.8003: *8* mungeAllFunctions
def mungeAllFunctions(self,aList):

    '''Scan for a '{' at the top level that is preceeded by ')' '''

    prevSemi = 0 # Previous semicolon: header contains all previous text
    i = 0
    firstOpen = None
    while i < len(aList):
        progress = i
        if self.is_string_or_comment(aList,i):
            j = self.skip_string_or_comment(aList,i)
            prevSemi = j
        elif self.match(aList,i,'('):
            if not firstOpen:
                firstOpen = i
            j = i + 1
        elif self.match(aList,i,'#'):
            # At this point, it is a preprocessor directive.
            j = self.skip_past_line(aList, i)
            prevSemi = j
        elif self.match(aList,i,';'):
            j = i + 1
            prevSemi = j
        elif self.match(aList,i,"{"):
            j = self.handlePossibleFunctionHeader(aList,i,prevSemi,firstOpen)
            prevSemi = j
            firstOpen = None # restart the scan
            # g.trace(repr(''.join(aList[prevSemi:prevSemi+20])))
        else:
            j = i + 1

        # Handle unusual cases.
        if j <= progress:
            j = progress + 1
        assert j > progress

        i = j
#@+node:ekr.20110916215321.8004: *9* handlePossibleFunctionHeader
# converts function header lines from c++ format to python format.
# That is, converts
# x1..nn w::y ( t1 z1,..tn zn) {
# to
# def y (z1,..zn): {

def handlePossibleFunctionHeader (self,aList,i,prevSemi,firstOpen):

    trace = False
    assert(self.match(aList,i,"{"))

    prevSemi = self.skip_ws_and_nl(aList, prevSemi)
    close = self.prevNonWsOrNlChar(aList,i)

    if close < 0 or aList[close] != ')':
        # Should not increase *Python* indent.
        return 1 + self.skip_to_matching_bracket(aList,i)

    if not firstOpen:
        return 1 + self.skip_to_matching_bracket(aList,i)

    close2 = self.skip_to_matching_bracket(aList, firstOpen)
    if close2 != close:
        return 1 + self.skip_to_matching_bracket(aList,i)

    open_paren = firstOpen
    assert(aList[open_paren]=='(')
    head = aList[prevSemi:open_paren]

    # do nothing if the head starts with "if", "for" or "while"
    k = self.skip_ws(head,0)
    if k >= len(head) or not head[k].isalpha():
        return 1 + self.skip_to_matching_bracket(aList,i)

    kk = self.skip_past_word(head,k)
    if kk > k:
        headString = ''.join(head[k:kk])
        # C keywords that might be followed by '{'
        # print "headString:", headString
        if headString in [ "class", "do", "for", "if", "struct", "switch", "while"]:
            return 1 + self.skip_to_matching_bracket(aList, i)

    args = aList[open_paren:close+1]
    k = 1 + self.skip_to_matching_bracket(aList,i)
    body = aList[close+1:k]

    if True and trace:
        g.trace('\nhead: %s\nargs: %s\nbody: %s' % (
            ''.join(head),''.join(args),''.join(body)))

    head = self.massageFunctionHead(head)
    args = self.massageFunctionArgs(args)
    body = self.massageFunctionBody(body)

    if False and trace:
        g.trace('\nhead2: %s\nargs2: %s\nbody2: %s' % (
            ''.join(head),''.join(args),''.join(body)))

    result = []
    if head: result.extend(head)
    if args: result.extend(args)
    if body: result.extend(body)

    aList[prevSemi:k] = result
    return prevSemi + len(result)
#@+node:ekr.20110916215321.8005: *9* massageFunctionArgs
def massageFunctionArgs (self,args):

    assert(args[0]=='(')
    assert(args[-1]==')')

    result = ['('] ; lastWord = []
    if self.class_name:
        for item in list("self,"): result.append(item) #can put extra comma

    i = 1
    while i < len(args):
        i = self.skip_ws_and_nl(args, i)
        c = args[i]
        if c.isalpha():
            j = self.skip_past_word(args,i)
            lastWord = args[i:j]
            i = j
        elif c == ',' or c == ')':
            for item in lastWord:
                result.append(item)
            if lastWord != [] and c == ',':
                result.append(',')
            lastWord = []
            i += 1
        else: i += 1
    if result[-1] == ',':
        del result[-1]
    result.append(')')
    result.append(':')
    # print "new args:", ''.join(result)
    return result
#@+node:ekr.20110916215321.8006: *9* massageFunctionHead (sets .class_name)
def massageFunctionHead (self,head):

    result = []
    prevWord = []
    self.class_name = ''
    i = 0
    # g.trace(repr(''.join(head)))
    while i < len(head):
        i = self.skip_ws_and_nl(head,i)
        if i < len(head) and head[i].isalpha():
            result = []
            j = self.skip_past_word(head,i)
            prevWord = head[i:j]
            i = j
            # look for ::word2
            i = self.skip_ws(head,i)
            if self.match(head,i,"::"):
                # Set the global to the class name.
                self.class_name = ''.join(prevWord)
                # print(class name:", self.class_name)
                i = self.skip_ws(head,i+2)
                if i < len(head) and (head[i]=='~' or head[i].isalpha()):
                    j = self.skip_past_word(head,i)
                    if head[i:j] == prevWord:
                        result.extend('__init__')
                    elif head[i]=='~' and head[i+1:j] == prevWord:
                        result.extend('__del__')
                    else:
                        # result.extend(list('::'))
                        result.extend(head[i:j])
                    i = j
            else:
                result.extend(prevWord)
        else: i += 1

    finalResult = list("def ")
    finalResult.extend(result)
    return finalResult
#@+node:ekr.20110916215321.8007: *9* massageFunctionBody & helpers
def massageFunctionBody (self,body):

    body = self.massageIvars(body)
    body = self.removeCasts(body)
    body = self.removeTypeNames(body)
    body = self.dedentBlocks(body)
    return body
#@+node:ekr.20110919224143.6928: *10* dedentBlocks
def dedentBlocks (self,body):

    '''Look for '{' preceded by '{' or '}' or ';'
    (with intervening whitespace and comments).
    '''

    i = 0
    while i < len(body):
        j = i
        ch = body[i]
        if self.is_string_or_comment(body,i):
            j = self.skip_string_or_comment(body,i)
        elif ch in '{};':
            # Look ahead ofr '{'
            j += 1
            while True:
                k = j
                j = self.skip_ws_and_nl(body,j)
                if self.is_string_or_comment(body,j):
                    j = self.skip_string_or_comment(body,j)
                if k == j: break
                assert k < j
            if self.match(body,j,'{'):
                k = j
                j = self.skip_to_matching_bracket(body,j)
                # g.trace('found block\n',''.join(body[k:j+1]))
                m = '# <Start dedented block>...'
                body[k:k+1] = list(m)
                j += len(m)
                while k < j:
                    progress = k
                    if body[k] == '\n':
                        k += 1
                        spaces = 0
                        while spaces < 4 and k < j:
                            if body[k] == ' ':
                                spaces += 1
                                k += 1
                            else:
                                break
                        if spaces > 0:
                            del body[k-spaces:k]
                            k -= spaces
                            j -= spaces
                    else:
                        k += 1
                    assert progress < k
                m = '    # <End dedented block>'
                body[j:j+1] = list(m)
                j += len(m)
        else:
            j = i + 1

        # Defensive programming.
        if i == j:
            j += 1
        assert i < j
        i = j

    return body
#@+node:ekr.20110916215321.8008: *10* massageIvars
def massageIvars (self,body):

    ivars = self.ivars_dict.get(self.class_name,[])
    i = 0
    while i < len(body):
        if self.is_string_or_comment(body,i):
            i = self.skip_string_or_comment(body,i)
        elif body[i].isalpha():
            j = self.skip_past_word(body,i)
            word = ''.join(body[i:j])
            # print "looking up:", word
            if word in ivars:
                # replace word by self.word
                # print "replacing", word, " by self.", word
                word = "self." + word
                word = list(word)
                body[i:j] = word
                delta = len(word)-(j-i)
                i = j + delta
            else: i = j
        else: i += 1
    return body
#@+node:ekr.20110916215321.8009: *10* removeCasts
def removeCasts (self,body):

    i = 0
    while i < len(body):
        if self.is_string_or_comment(body,i):
            i = self.skip_string_or_comment(body,i)
        elif self.match(body, i, '('):
            start = i
            i = self.skip_ws(body, i+1)
            if body[i].isalpha():
                j = self.skip_past_word(body,i)
                word = ''.join(body[i:j])
                i = j
                if word in self.class_list or word in self.type_list:
                    i = self.skip_ws(body, i)
                    while self.match(body,i,'*'):
                        i += 1
                    i = self.skip_ws(body, i)
                    if self.match(body,i,')'):
                        i += 1
                        # print "removing cast:", ''.join(body[start:i])
                        del body[start:i]
                        i = start
        else: i += 1
    return body
#@+node:ekr.20110916215321.8010: *10* removeTypeNames
# Do _not_ remove type names when preceeded by new.

def removeTypeNames (self,body):

    i = 0
    while i < len(body):
        if self.is_string_or_comment(body,i):
            i = self.skip_string_or_comment(body,i)
        elif self.match_word(body, i, "new"):
            i = self.skip_past_word(body,i)
            i = self.skip_ws(body,i)
            # don't remove what follows new.
            if body[i].isalpha():
                i = self.skip_past_word(body,i)
        elif body[i].isalpha():
            j = self.skip_past_word(body,i)
            word = ''.join(body[i:j])
            if word in self.class_list or word in self.type_list:
                j = self.skip_ws(body,j)
                while self.match(body,j,'*'):
                    j += 1
                # print "Deleting type name:", ''.join(body[i:j])
                j = self.skip_ws(body,j)
                del body[i:j]
            else:
                i = j
        else: i += 1
    return body
#@+node:ekr.20121016093159.10185: *6* << class TS_To_Python (To_Python) >>
class TS_To_Python (To_Python):

    @others
#@+node:ekr.20121016093159.10297: *7* ctor (TS_To_Python)
def __init__ (self,c):

    c.editCommands.To_Python.__init__(self,c)
        # init the base class

    self.class_name = ''
        # The class name for the present function.  Used to modify ivars.
#@+node:ekr.20121015183335.10145: *7* convertCodeList (TS_To_Python) & helpers
def convertCodeList(self,aList):

    r,sr = self.replace,self.safe_replace

    # First...
    r(aList, '\r', '')
    self.mungeAllFunctions(aList)
    self.mungeAllClasses(aList)

    # Second...
    sr(aList, ' -> ', '.')
    sr(aList, '->', '.')
    sr(aList, ' . ', '.')
    # sr(aList, 'this.self', 'self')
    sr(aList, '{', '')
    sr(aList, '}', '')
    sr(aList, 'else if', 'elif')
    sr(aList, 'else', 'else:')
    sr(aList, '&&', ' and ')
    sr(aList, '||', ' or ')
    sr(aList, 'true', 'True')
    sr(aList, 'false', 'False')
    sr(aList, 'null', 'None')
    sr(aList, 'this', 'self')
    sr(aList, 'try', 'try:')
    sr(aList, 'catch', 'except:')
    sr(aList, 'constructor', '__init__')
    sr(aList, 'new ','')
    # sr(aList, 'var ','')
        # var usually indicates something weird, or an uninited var,
        # so it may be good to retain as a marker.

    # Third...
    self.handle_all_keywords(aList)
    self.insert_not(aList)
    self.removeSemicolonsAtEndOfLines(aList)
        # after processing for keywords
    self.comment_scope_ids(aList)

    # Last...
    self.removeBlankLines(aList)
    self.removeExcessWs(aList)
    # I usually don't like extra whitespace. YMMV.
    sr(aList, '  and ', ' and ')
    sr(aList, '  not ', ' not ')
    sr(aList, '  or ',  ' or ')
    sr(aList, ' and  ', ' and ')
    sr(aList, ' not  ', ' not ')
    sr(aList, ' or  ',  ' or ')
    sr(aList, ' :', ':') 
    sr(aList, ', ', ',')
    sr(aList, ' ,', ',')
    sr(aList, ' (', '(')
    sr(aList, '( ', '(')
    sr(aList, ' )', ')')
    sr(aList, ') ', ')')
    sr(aList, ' and(', ' and (')
    sr(aList, ' not(', ' not (')
    sr(aList, ' or(',  ' or (')
    sr(aList, ')and ', ') and ')
    sr(aList, ')not ', ') not ')
    sr(aList, ')or ',  ') or ')
    sr(aList, ')and(', ') and (')
    sr(aList, ')not(', ') not (')
    sr(aList, ')or(',  ') or (')
    sr(aList, '@language javascript','@language python')
    self.replaceComments(aList) # should follow all calls to safe_replace
    self.removeTrailingWs(aList)
    r(aList, '\t ', '\t') # happens when deleting declarations.
#@+node:ekr.20121015183335.10191: *8* comment_scope_ids
def comment_scope_ids (self,aList):

    '''convert (public|private|export) aLine to aLine # (public|private|export)'''

    scope_ids = ('public','private','export',)
    i = 0
    if any([self.match_word(aList,i,z) for z in scope_ids]):
        i = self.handle_scope_keyword(aList,i)
    while i < len(aList):
        progress = i
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif aList[i] == '\n':
            i += 1
            i = self.skip_ws(aList,i)
            if any([self.match_word(aList,i,z) for z in scope_ids]):
                i = self.handle_scope_keyword(aList,i)
        else:
            i += 1
        assert i > progress
    # print "handAllKeywords2:", ''.join(aList)
#@+node:ekr.20121015183335.10193: *9* handle_scope_keyword
def handle_scope_keyword (self,aList,i):

    i1 = i
    for word in ('public','private','export'):
        if self.match_word(aList,i,word):
            i += len(word)
            break
    else:
        assert False,'not a scope id: %s' % word

    # Skip any following spaces.
    i2 = self.skip_ws(aList,i)

    # Scan to the next newline:
    i3 = self.skip_line(aList,i)

    # Optional: move the word to a trailing comment.
    comment = list(' # %s' % word) if False else []

    # Change the list in place.
    aList[i1:i3] = aList[i2:i3] + comment
    i = i1 + (i3-i2) + len(comment)
    # g.trace(''.join(aList[i1:i]))
    return i
#@+node:ekr.20121015183335.10157: *8* handle_all_keywords
def handle_all_keywords (self,aList):

    '''
    converts if ( x ) to if x:
    converts while ( x ) to while x:
    '''

    statements = ('elif','for','if','while',)
    i = 0
    while i < len(aList):
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif any([self.match_word(aList,i,z) for z in statements]):
            i = self.handle_keyword(aList,i)
        # elif (
            # self.match_word(aList,i,"if") or
            # self.match_word(aList,i,"while") or
            # self.match_word(aList,i,"for") or
            # self.match_word(aList,i,"elif")
        # ):
            # i = self.handle_keyword(aList,i)
        else:
            i += 1
    # print "handAllKeywords2:", ''.join(aList)
#@+node:ekr.20121015183335.10158: *9* handle_keyword
def handle_keyword (self,aList,i):

    if self.match_word(aList,i,"if"):
        i += 2
    elif self.match_word(aList,i,"elif"):
        i += 4
    elif self.match_word(aList,i,"while"):
        i += 5
    elif self.match_word(aList,i,"for"):
        i += 3
    else: assert False,'not a keyword'

    # Make sure one space follows the keyword.
    k = i
    i = self.skip_ws(aList,i)
    if k == i:
        c = aList[i]
        aList[i:i+1] = [ ' ', c ]
        i += 1

    # Remove '(' and matching ')' and add a ':'
    if aList[i] == "(":
        # Look ahead.  Don't remove if we span a line.
        j = self.skip_to_matching_bracket(aList, i)
        k = i
        found = False
        while k < j and not found:
            found = aList[k] == '\n'
            k += 1
        if not found:
            j = self.removeMatchingBrackets(aList,i)
        if j > i and j < len(aList):
            ch = aList[j]
            aList[j:j+1] = [ch,":", " "]
            j = j + 2
        return j
    return i
#@+node:ekr.20121016024338.10190: *8* mungeAllClasses
def mungeAllClasses(self,aList):

    '''Scan for a '{' at the top level that is preceeded by ')' '''

    i = 0
    while i < len(aList):
        progress = i
        if self.is_string_or_comment(aList,i):
            i = self.skip_string_or_comment(aList,i)
        elif self.match_word(aList,i,'class'):
            i1 = i
            i = self.skip_line(aList,i)
            aList[i-1:i] = list('%s:' % aList[i-1])
            s = ''.join(aList[i1:i])
            k = s.find(' extends ')
            if k > -1:
                k1 = k
                k = g.skip_id(s,k+1)
                k = g.skip_ws(s,k)
                if k < len(s) and g.is_c_id(s[k]):
                    k2 = g.skip_id(s,k)
                    word = s[k:k2]
                    aList[i1:i] = list('%s (%s)' % (s[:k1],word))

        elif self.match_word(aList,i,'interface'):
            aList[i:i+len('interface')] = list('class')
            i = self.skip_line(aList,i)
            aList[i-1:i] = list('%s: # interface' % aList[i-1])
            i = self.skip_line(aList,i) # Essential.
        else:
            i += 1
        assert i > progress
#@+node:ekr.20121015183335.10148: *8* mungeAllFunctions & helpers
def mungeAllFunctions(self,aList):

    '''Scan for a '{' at the top level that is preceeded by ')' '''

    prevSemi = 0 # Previous semicolon: header contains all previous text
    i = 0
    firstOpen = None
    while i < len(aList):
        progress = i
        if self.is_string_or_comment(aList,i):
            j = self.skip_string_or_comment(aList,i)
            prevSemi = j
        elif self.match(aList,i,'('):
            if not firstOpen:
                firstOpen = i
            j = i + 1
        elif self.match(aList,i,';'):
            j = i + 1
            prevSemi = j
        elif self.match(aList,i,"{"):
            j = self.handlePossibleFunctionHeader(aList,i,prevSemi,firstOpen)
            prevSemi = j
            firstOpen = None # restart the scan
            # g.trace(repr(''.join(aList[prevSemi:prevSemi+20])))
        else:
            j = i + 1

        # Handle unusual cases.
        if j <= progress:
            j = progress + 1
        assert j > progress

        i = j
#@+node:ekr.20121015183335.10149: *9* handlePossibleFunctionHeader
# converts function header lines from typescript format to python format.
# That is, converts
### x1..nn w::y ( t1 z1,..tn zn) { C++
# (public|private|export) name (t1: z1, ... tn: zn {
# to
# def y (z1,..zn): { # (public|private|export)

def handlePossibleFunctionHeader (self,aList,i,prevSemi,firstOpen):

    trace = False
    assert(self.match(aList,i,"{"))

    prevSemi = self.skip_ws_and_nl(aList, prevSemi)
    close = self.prevNonWsOrNlChar(aList,i)

    if close < 0 or aList[close] != ')':
        # Should not increase *Python* indent.
        return 1 + self.skip_to_matching_bracket(aList,i)

    if not firstOpen:
        return 1 + self.skip_to_matching_bracket(aList,i)

    close2 = self.skip_to_matching_bracket(aList, firstOpen)
    if close2 != close:
        return 1 + self.skip_to_matching_bracket(aList,i)

    open_paren = firstOpen
    assert(aList[open_paren]=='(')
    head = aList[prevSemi:open_paren]

    # do nothing if the head starts with "if", "for" or "while"
    k = self.skip_ws(head,0)
    if k >= len(head) or not head[k].isalpha():
        return 1 + self.skip_to_matching_bracket(aList,i)

    kk = self.skip_past_word(head,k)
    if kk > k:
        headString = ''.join(head[k:kk])
        # C keywords that might be followed by '{'
        # print "headString:", headString
        if headString in [ "do", "for", "if", "struct", "switch", "while"]:
            return 1 + self.skip_to_matching_bracket(aList, i)

    args = aList[open_paren:close+1]
    k = 1 + self.skip_to_matching_bracket(aList,i)
    body = aList[close+1:k]

    if trace:
        g.trace('\nhead: %s\nargs: %s\nbody: %s' % (
            ''.join(head),''.join(args),''.join(body)))

    head = self.massageFunctionHead(head)
    args = self.massageFunctionArgs(args)
    body = self.massageFunctionBody(body)

    if False and trace:
        g.trace('\nhead2: %s\nargs2: %s\nbody2: %s' % (
            ''.join(head),''.join(args),''.join(body)))

    result = []
    if head: result.extend(head)
    if args: result.extend(args)
    if body: result.extend(body)

    aList[prevSemi:k] = result
    return prevSemi + len(result)
#@+node:ekr.20121015183335.10150: *9* massageFunctionArgs
def massageFunctionArgs (self,args):

    assert(args[0]=='(')
    assert(args[-1]==')')

    result = ['('] ; lastWord = []
    if self.class_name:
        for item in list("self,"): result.append(item) #can put extra comma

    i = 1
    while i < len(args):
        i = self.skip_ws_and_nl(args, i)
        c = args[i]
        if c.isalpha():
            j = self.skip_past_word(args,i)
            lastWord = args[i:j]
            i = j
        elif c == ',' or c == ')':
            for item in lastWord:
                result.append(item)
            if lastWord != [] and c == ',':
                result.append(',')
            lastWord = []
            i += 1
        else: i += 1
    if result[-1] == ',':
        del result[-1]
    result.append(')')
    result.append(':')
    # print "new args:", ''.join(result)
    return result
#@+node:ekr.20121015183335.10151: *9* massageFunctionHead (sets .class_name)
def massageFunctionHead (self,head):

    result = []
    prevWord = []
    self.class_name = ''
    i = 0
    # g.trace(repr(''.join(head)))
    while i < len(head):
        i = self.skip_ws_and_nl(head,i)
        if i < len(head) and head[i].isalpha():
            result = []
            j = self.skip_past_word(head,i)
            prevWord = head[i:j]
            i = j
            # look for ::word2
            i = self.skip_ws(head,i)
            if self.match(head,i,"::"):
                # Set the global to the class name.
                self.class_name = ''.join(prevWord)
                # print(class name:", self.class_name)
                i = self.skip_ws(head,i+2)
                if i < len(head) and (head[i]=='~' or head[i].isalpha()):
                    j = self.skip_past_word(head,i)
                    if head[i:j] == prevWord:
                        result.extend('__init__')
                    elif head[i]=='~' and head[i+1:j] == prevWord:
                        result.extend('__del__')
                    else:
                        # result.extend(list('::'))
                        result.extend(head[i:j])
                    i = j
            else:
                result.extend(prevWord)
        else: i += 1

    finalResult = list("def ")
    finalResult.extend(result)
    return finalResult
#@+node:ekr.20121015183335.10152: *9* massageFunctionBody & helper
def massageFunctionBody (self,body):

    # body = self.massageIvars(body)
    # body = self.removeCasts(body)
    # body = self.removeTypeNames(body)
    body = self.dedentBlocks(body)
    return body
#@+node:ekr.20121015183335.10153: *10* dedentBlocks
def dedentBlocks (self,body):

    '''Look for '{' preceded by '{' or '}' or ';'
    (with intervening whitespace and comments).
    '''

    i = 0
    while i < len(body):
        j = i
        ch = body[i]
        if self.is_string_or_comment(body,i):
            j = self.skip_string_or_comment(body,i)
        elif ch in '{};':
            # Look ahead ofr '{'
            j += 1
            while True:
                k = j
                j = self.skip_ws_and_nl(body,j)
                if self.is_string_or_comment(body,j):
                    j = self.skip_string_or_comment(body,j)
                if k == j: break
                assert k < j
            if self.match(body,j,'{'):
                k = j
                j = self.skip_to_matching_bracket(body,j)
                # g.trace('found block\n',''.join(body[k:j+1]))
                m = '# <Start dedented block>...'
                body[k:k+1] = list(m)
                j += len(m)
                while k < j:
                    progress = k
                    if body[k] == '\n':
                        k += 1
                        spaces = 0
                        while spaces < 4 and k < j:
                            if body[k] == ' ':
                                spaces += 1
                                k += 1
                            else:
                                break
                        if spaces > 0:
                            del body[k-spaces:k]
                            k -= spaces
                            j -= spaces
                    else:
                        k += 1
                    assert progress < k
                m = '    # <End dedented block>'
                body[j:j+1] = list(m)
                j += len(m)
        else:
            j = i + 1

        # Defensive programming.
        if i == j:
            j += 1
        assert i < j
        i = j

    return body
#@+node:ekr.20100209160132.5763: *5* cache (leoEditCommands)
def clearAllCaches (self,event=None):

    '''Clear all of Leo's file caches.'''

    c = self.c
    if c.cacher:
        c.cacher.clearAllCaches()

def clearCache (self,event=None):

    '''Clear the outline's file cache.'''

    c = self.c
    if c.cacher:
        c.cacher.clearCache()
#@+node:ekr.20050920084036.57: *5* capitalization & case
#@+node:ekr.20051015114221: *6* capitalizeWord & up/downCaseWord
def capitalizeWord (self,event):
    '''Capitalize the word at the cursor.'''
    self.capitalizeHelper(event,'cap','capitalize-word')

def downCaseWord (self,event):
    '''Convert all characters of the word at the cursor to lower case.'''
    self.capitalizeHelper(event,'low','downcase-word')

def upCaseWord (self,event):
    '''Convert all characters of the word at the cursor to UPPER CASE.'''
    self.capitalizeHelper(event,'up','upcase-word')
#@+node:ekr.20050920084036.145: *6* changePreviousWord (not used)
# def changePreviousWord (self,event):

    # k = self.k ; stroke = k.stroke
    # w = self.editWidget(event)
    # if not w: return

    # i = w.getInsertPoint()
    # self.beginCommand(undoType='change-previous-word')
    # self.moveWordHelper(event,extend=False,forward=False)

    # if stroke == '<Alt-c>':
        # self.capitalizeWord(event)
    # elif stroke == '<Alt-u>':
        # self.upCaseWord(event)
    # elif stroke == '<Alt-l>':
        # self.downCaseWord(event)

    # w.setInsertPoint(i)

    # self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20051015114221.1: *6* capitalizeHelper
def capitalizeHelper (self,event,which,undoType):

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getWord(s,ins)
    word = s[i:j]
    # g.trace('word',repr(word))
    if not word.strip(): return

    self.beginCommand(undoType=undoType)

    if   which == 'cap':  word2 = word.capitalize()
    elif which == 'low':  word2 = word.lower()
    elif which == 'up':   word2 = word.upper()
    else: g.trace('can not happen: which = %s' %s (which))

    changed = word != word2
    # g.trace('changed',changed,'word2',repr(word2))

    if changed:
        w.delete(i,j)
        w.insert(i,word2)
        w.setSelectionRange(ins,ins,insert=ins)

    self.endCommand(changed=changed,setLabel=True)
#@+node:ekr.20051022142249: *5* clicks and focus (EditCommandsClass)
#@+node:ekr.20060211100905: *6* activate-x-menu & activateMenu (EditCommandsClass)
def activateCmdsMenu    (self,event=None):
    '''Activate Leo's Cmnds menu.'''
    self.activateMenu('Cmds')

def activateEditMenu    (self,event=None):
    '''Activate Leo's Edit menu.'''
    self.activateMenu('Edit')

def activateFileMenu    (self,event=None):
    '''Activate Leo's File menu.'''
    self.activateMenu('File')

def activateHelpMenu    (self,event=None):
    '''Activate Leo's Help menu.'''
    self.activateMenu('Help')

def activateOutlineMenu (self,event=None):
    '''Activate Leo's Outline menu.'''
    self.activateMenu('Outline')

def activatePluginsMenu (self,event=None):
    '''Activate Leo's Plugins menu.'''
    self.activateMenu('Plugins')

def activateWindowMenu  (self,event=None):
    '''Activate Leo's Window menu.'''
    self.activateMenu('Window')

def activateMenu (self,menuName):
    c = self.c
    c.frame.menu.activateMenu(menuName)
#@+node:ekr.20051022144825.1: *6* cycleFocus
def cycleFocus (self,event):
    '''Cycle the keyboard focus between Leo's outline, body and log panes.'''
    c = self.c ; k = c.k ; w = event and event.widget
    body = c.frame.body.wrapper
    log  = c.frame.log.logCtrl
    tree = c.frame.tree.canvas
    # A hack for the Qt gui.
    if hasattr(w,'logCtrl'):
        w = w.logCtrl
    panes = [body,log,tree]
    # g.trace(w in panes,event.widget,panes)
    if w in panes:
        i = panes.index(w) + 1
        if i >= len(panes): i = 0
        pane = panes[i]
    else:
        pane = body
    # Warning: traces mess up the focus
    # g.pr(g.app.gui.widget_name(w),g.app.gui.widget_name(pane))
    # This works from the minibuffer *only* if there is no typing completion.
    c.widgetWantsFocusNow(pane)
    k.newMinibufferWidget = pane
    k.showStateAndMode()
#@+node:ekr.20060613090701: *6* cycleAllFocus (EditCommandsClass)
editWidgetCount = 0

def cycleAllFocus (self,event):

    '''Cycle the keyboard focus between Leo's outline,
    all body editors and all tabs in the log pane.'''

    trace = False and not g.unitTesting
    c,k = self.c,self.c.k
    w = event and event.widget # Does **not** require a text widget.
    pane = None # The widget that will get the new focus.
    log = c.frame.log
    w_name = g.app.gui.widget_name
    if trace: g.trace('**before',w_name(w),'isLog',log.isLogWidget(w))
    # w may not be the present body widget, so test its name, not its id.
    if w_name(w).find('tree') > -1 or w_name(w).startswith('head'):
        pane = c.frame.body.wrapper
    elif w_name(w).startswith('body'):
        # Cycle through the *body* editor if there are several.
        n = c.frame.body.numberOfEditors
        if n > 1:
            self.editWidgetCount += 1
            if self.editWidgetCount == 1:
                pane = c.frame.body.wrapper
            elif self.editWidgetCount > n:
                self.editWidgetCount = 0
                c.frame.log.selectTab('Log')
                pane = c.frame.log.logCtrl
            else:
                c.frame.body.cycleEditorFocus(event)
                pane = None
        else:
            self.editWidgetCount = 0
            c.frame.log.selectTab('Log')
            pane = c.frame.log.logCtrl
    elif log.isLogWidget(w):
        # A log widget.  Cycle until we come back to 'Log'.
        log.cycleTabFocus()
        pane = c.frame.tree.canvas if log.tabName == 'Log' else None
    else:
        # A safe default: go to the body.
        if trace: g.trace('* default to body')
        pane = c.frame.body.wrapper
    if trace: g.trace('**after',w_name(pane),pane)
    if pane:
        k.newMinibufferWidget = pane
        c.widgetWantsFocusNow(pane)
        k.showStateAndMode()
#@+node:ekr.20051022144825: *6* focusTo...
def focusToBody (self,event=None):
    '''Put the keyboard focus in Leo's body pane.'''
    c = self.c ; k = c.k
    c.bodyWantsFocus()
    if k:
        k.setDefaultInputState()
        k.showStateAndMode()

def focusToLog (self,event=None):
    '''Put the keyboard focus in Leo's log pane.'''
    self.c.logWantsFocus()

def focusToMinibuffer (self,event=None):
    '''Put the keyboard focus in Leo's minibuffer.'''
    self.c.minibufferWantsFocus()

def focusToTree (self,event=None):
    '''Put the keyboard focus in Leo's outline pane.'''
    self.c.treeWantsFocus()
#@+node:ekr.20060211063744.1: *6* clicks in the headline (leoEditCommands)
# These call wrappers that trigger hooks.

def clickHeadline (self,event=None):
    '''Simulate a click in the headline of the presently selected node.'''
    c = self.c
    c.frame.tree.onHeadlineClick(event,c.p)

def doubleClickHeadline (self,event=None):
    '''Simulate a double click in headline of the presently selected node.'''
    c = self.c
    return c.frame.tree.onDoubleClickHeadline(event,c.p)

# This is not used in Leo at present.

def rightClickHeadline (self,event=None):
    '''Simulate a right click in the headline of the presently selected node.'''
    c = self.c
    c.frame.tree.onHeadlineRightClick(event,c.p)
#@+node:ekr.20060211055455: *6* clicks in the icon box (leoEditCommands)
# These call the actual event handlers so as to trigger hooks.

def ctrlClickIconBox(self,event=None):
    '''Simulate a ctrl-click in the icon box of the presently selected node.'''
    c = self.c
    c.frame.tree.OnIconCtrlClick(c.p)
        # Calls the base LeoTree method.

def clickIconBox (self,event=None):
    '''Simulate a click in the icon box of the presently selected node.'''
    c = self.c ; p = c.p
    c.frame.tree.onIconBoxClick(event,p=p)

def doubleClickIconBox (self,event=None):
    '''Simulate a double-click in the icon box of the presently selected node.'''
    c = self.c ; p = c.p
    c.frame.tree.onIconBoxDoubleClick(event,p=p)

def rightClickIconBox (self,event=None):

    '''Simulate a right click in the icon box of the presently selected node.'''
    c = self.c ; p = c.p
    c.frame.tree.onIconBoxRightClick(event,p=p)
#@+node:ekr.20060211062025: *6* clickClickBox
# Call the actual event handlers so as to trigger hooks.

def clickClickBox (self,event=None):

    '''Simulate a click in the click box (+- box) of the presently selected node.'''

    c = self.c ; p = c.p
    c.frame.tree.onClickBoxClick(event,p=p)
#@+node:ekr.20060211063744.2: *6* simulate...Drag
# These call the drag setup methods which in turn trigger hooks.

def simulateBeginDrag (self,event=None):

    '''Simulate the start of a drag in the presently selected node.'''
    c = self.c ; p = c.p
    c.frame.tree.startDrag(event,p=p)

def simulateEndDrag (self,event=None):

    '''Simulate the end of a drag in the presently selected node.'''
    c = self.c

    # Note: this assumes that tree.startDrag has already been called.
    c.frame.tree.endDrag(event)
#@+node:ekr.20051019183105: *5* color & font
#@+node:ekr.20051019183105.1: *6* show-colors
def showColors (self,event):

    '''Open a tab in the log pane showing various color pickers.'''

    c = self.c ; log = c.frame.log ; tabName = 'Colors'

    if log.frameDict.get(tabName):
        log.selectTab(tabName)
    else:
        log.selectTab(tabName)
        log.createColorPicker(tabName)
#@+node:ekr.20051019201809: *6* editCommands.show-fonts & helpers
def showFonts (self,event):

    '''Open a tab in the log pane showing a font picker.'''

    c = self.c ; log = c.frame.log ; tabName = 'Fonts'

    if log.frameDict.get(tabName):
        log.selectTab(tabName)
    else:
        log.selectTab(tabName)
        log.createFontPicker(tabName)
#@+node:ekr.20050920084036.132: *5* comment column...
#@+node:ekr.20050920084036.133: *6* setCommentColumn
def setCommentColumn (self,event):

    '''Set the comment column for the indent-to-comment-column command.'''

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    ins = w.getInsertPoint()
    row,col = g.convertPythonIndexToRowCol(s,ins)
    self.ccolumn = col
#@+node:ekr.20050920084036.134: *6* indentToCommentColumn
def indentToCommentColumn (self,event):

    '''Insert whitespace to indent the line containing the insert point to the comment column.'''

    w = self.editWidget(event)
    if not w: return
    self.beginCommand(undoType='indent-to-comment-column')
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    line = s[i:j]
    c1 = int(self.ccolumn)
    line2 = ' ' * c1 + line.lstrip()
    if line2 != line:
        w.delete(i,j)
        w.insert(i,line2)
    w.setInsertPoint(i+c1)
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.62: *5* esc methods for Python evaluation
#@+node:ekr.20050920084036.63: *6* watchEscape
def watchEscape (self,event):

    '''Enter watch escape mode.'''

    c,k = self.c,self.k

    char = event and event.char or ''

    if not k.inState():
        k.setState('escape','start',handler=self.watchEscape)
        k.setLabelBlue('Esc ')
    elif k.getStateKind() == 'escape':
        state = k.getState('escape')
        # hi1 = k.keysymHistory [0]
        # hi2 = k.keysymHistory [1]
        data1 = g.app.lossage[0]
        data2 = g.app.lossage[1]
        ch1, stroke1 = data1
        ch2, stroke2 = data2

        if state == 'esc esc' and char == ':':
            self.evalExpression(event)
        elif state == 'evaluate':
            self.escEvaluate(event)
        # elif hi1 == hi2 == 'Escape':
        elif stroke1 == 'Escape' and stroke2 == 'Escape':
            k.setState('escape','esc esc')
            k.setLabel('Esc Esc -')
        elif char not in ('Shift_L','Shift_R'):
            k.keyboardQuit()
#@+node:ekr.20050920084036.64: *6* escEvaluate (Revise)
def escEvaluate (self,event):

    c,k = self.c,self.k

    w = self.editWidget(event)
    if not w: return

    char = event and event.char or ''

    if k.getLabel() == 'Eval:':
        k.setLabel('')

    if char in ('\n','Return'):
        expression = k.getLabel()
        try:
            ok = False
            result = eval(expression,{},{})
            result = str(result)
            i = w.getInsertPoint()
            w.insert(i,result)
            ok = True
        finally:
            k.keyboardQuit()
            if not ok:
                k.setStatusLabel('Error: Invalid Expression')
    else:
        k.updateLabel(event)
#@+node:ekr.20050920084036.65: *5* evalExpression
def evalExpression (self,event):
    '''Evaluate a Python Expression entered in the minibuffer.'''
    k = self.k ; state = k.getState('eval-expression')
    if state == 0:
        k.setLabelBlue('Eval: ')
        k.getArg(event,'eval-expression',1,self.evalExpression)
    else:
        k.clearState()
        try:
            e = k.arg
            result = str(eval(e,{},{}))
            k.setLabelGrey('Eval: %s -> %s' % (e,result))
        except Exception:
            k.setLabelGrey('Invalid Expression: %s' % e)
#@+node:ekr.20050920084036.66: *5* fill column and centering
@
These methods are currently just used in tandem to center the line or region within the fill column.
for example, dependent upon the fill column, this text:

cats
raaaaaaaaaaaats
mats
zaaaaaaaaap

may look like

                                 cats
                           raaaaaaaaaaaats
                                 mats
                             zaaaaaaaaap

after an center-region command via Alt-x.
@c

@others
#@+node:ekr.20050920084036.67: *6* centerLine
def centerLine (self,event):

    '''Centers line within current fill column'''

    c,k,w = self.c,self.k,self.editWidget(event)
    if not w: return

    if self.fillColumn > 0:
        fillColumn = self.fillColumn
    else:
        d = c.scanAllDirectives()
        fillColumn = d.get("pagewidth")

    s = w.getAllText()
    i,j = g.getLine(s,w.getInsertPoint())
    line = s [i:j].strip()
    if not line or len(line) >= fillColumn: return

    self.beginCommand(undoType='center-line')
    n = (fillColumn-len(line)) / 2
    ws = ' ' * n
    k = g.skip_ws(s,i)
    if k > i: w.delete(i,k-i)
    w.insert(i,ws)
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.68: *6* setFillColumn
def setFillColumn (self,event):

    '''Set the fill column used by the center-line and center-region commands.'''

    k = self.k ; state = k.getState('set-fill-column')

    if state == 0:
        k.setLabelBlue('Set Fill Column: ')
        k.getArg(event,'set-fill-column',1,self.setFillColumn)
    else:
        k.clearState()
        try:
            # Bug fix: 2011/05/23: set the fillColumn ivar!
            self.fillColumn = n = int(k.arg)
            k.setLabelGrey('fill column is: %d' % n)
            k.commandName = 'set-fill-column %d' % n
        except ValueError:
            k.resetLabel()
#@+node:ekr.20050920084036.69: *6* centerRegion
def centerRegion (self,event):

    '''Centers the selected text within the fill column'''

    c,k,w = self.c,self.k,self.editWidget(event)
    if not w: return

    s = w.getAllText()
    sel_1, sel_2 = w.getSelectionRange()
    ind, junk = g.getLine(s,sel_1)
    junk, end = g.getLine(s,sel_2)

    if self.fillColumn > 0:
        fillColumn = self.fillColumn
    else:
        d = c.scanAllDirectives()
        fillColumn = d.get("pagewidth")

    self.beginCommand(undoType='center-region')

    inserted = 0
    while ind < end:
        s = w.getAllText()
        i, j = g.getLine(s,ind)
        line = s [i:j].strip()
        # g.trace(len(line),repr(line))
        if len(line) >= fillColumn:
            ind = j
        else:
            n = int((fillColumn-len(line))/2)
            inserted += n
            k = g.skip_ws(s,i)
            if k > i: w.delete(i,k-i)
            w.insert(i,' '*n)
            ind = j + n-(k-i)

    w.setSelectionRange(sel_1,sel_2+inserted)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.70: *6* setFillPrefix
def setFillPrefix( self, event ):

    '''Make the selected text the fill prefix.'''

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    i,j = w.getSelectionRange()
    self.fillPrefix = s[i:j]
#@+node:ekr.20050920084036.71: *6* _addPrefix
def _addPrefix (self,ntxt):

    # pylint: disable=deprecated-lambda
    ntxt = ntxt.split('.')
    ntxt = map(lambda a: self.fillPrefix+a,ntxt)
    ntxt = '.'.join(ntxt)
    return ntxt
#@+node:ekr.20060417194232: *5* find (quick)
#@+node:ekr.20060925151926: *6* backward/findCharacter & helper
def backwardFindCharacter (self,event):
    '''Search backwards for a character.'''
    return self.findCharacterHelper(event,backward=True,extend=False)

def backwardFindCharacterExtendSelection (self,event):
    '''Search backward for a character, extending the selection.'''
    return self.findCharacterHelper(event,backward=True,extend=True)

def findCharacter (self,event):
    '''Search for a character.'''
    return self.findCharacterHelper(event,backward=False,extend=False)

def findCharacterExtendSelection (self,event):
    '''Search for a character, extending the selection.'''
    return self.findCharacterHelper(event,backward=False,extend=True)
#@+node:ekr.20060417194232.1: *7* findCharacterHelper
def findCharacterHelper (self,event,backward,extend):
    '''Put the cursor at the next occurance of a character on a line.'''
    c = self.c ; k = c.k ; tag = 'find-char' ; state = k.getState(tag)
    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        self.event = event
        self.backward = backward
        self.extend = extend or self.extendMode # Bug fix: 2010/01/19
        self.insert = w.getInsertPoint()
        s = '%s character%s: ' % (
            'Backward find' if backward else 'Find',
            ' & extend' if extend else '')
        k.setLabelBlue(s)
        # Get the arg without touching the focus.
        k.getArg(event,tag,1,self.findCharacter,oneCharacter=True,useMinibuffer=False)
    else:
        event = self.event ; w = self.w
        backward = self.backward
        extend = self.extend or self.extendMode
        ch = k.arg ; s = w.getAllText()
        ins = w.toPythonIndex(self.insert)
        i = ins + -1 if backward else +1 # skip the present character.
        if backward:
            start = 0
            j = s.rfind(ch,start,max(start,i)) # Skip the character at the cursor.
            if j > -1: self.moveToHelper(event,j,extend)
        else:
            end = len(s)
            j = s.find(ch,min(i,end),end) # Skip the character at the cursor.
            if j > -1: self.moveToHelper(event,j,extend)
        k.resetLabel()
        k.clearState()
#@+node:ekr.20060417194232.2: *6* findWord and FindWordOnLine & helper
def findWord(self,event):

    '''Put the cursor at the next word that starts with a character.'''

    return self.findWordHelper(event,oneLine=False)

def findWordInLine(self,event):

    '''Put the cursor at the next word (on a line) that starts with a character.'''

    return self.findWordHelper(event,oneLine=True)

#@+node:ekr.20080408060320.1: *7* findWordHelper
def findWordHelper (self,event,oneLine):

    k = self.k ; tag = 'find-word' ; state = k.getState(tag)

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        self.oneLineFlag = oneLine
        k.setLabelBlue('Find word %sstarting with: ' % (
            'in line ' if oneLine else ''))
        k.getArg(event,tag,1,self.findWord,oneCharacter=True)
    else:        
        ch = k.arg
        if ch:
            w = self.w
            i = w.getInsertPoint()
            s = w.getAllText()
            end = len(s)
            if self.oneLineFlag:
                end = s.find('\n',i) # Limit searches to this line.
                if end == -1: end = len(s)
            while i < end:
                i = s.find(ch,i+1,end) # Ensure progress and i > 0.
                if i == -1:
                    break
                elif not g.isWordChar(s[i-1]):
                    w.setSelectionRange(i,i,insert=i)
                    break
        k.resetLabel()
        k.clearState()
#@+node:ekr.20050920084036.72: *5* goto...
#@+node:ekr.20050929115226: *6* gotoCharacter
def gotoCharacter (self,event):

    '''Put the cursor at the n'th character of the buffer.'''

    k = self.k ; state = k.getState('goto-char')

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        k.setLabelBlue("Goto n'th character: ")
        k.getArg(event,'goto-char',1,self.gotoCharacter)
    else:
        n = k.arg ; w = self.w ; ok = False
        if n.isdigit():
            n = int(n)
            if n >= 0:
                w.setInsertPoint(n)
                w.seeInsertPoint()
                ok = True
        if not ok:
            g.warning('goto-char takes non-negative integer argument')
        k.resetLabel()
        k.clearState()
#@+node:ekr.20060417181052: *6* gotoGlobalLine
def gotoGlobalLine (self,event):

    '''Put the cursor at the n'th line of a file or script.
    This is a minibuffer interface to Leo's legacy Go To Line number command.'''

    c = self.c
    k = self.k ; tag = 'goto-global-line' ; state = k.getState(tag)

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        k.setLabelBlue('Goto global line: ')
        k.getArg(event,tag,1,self.gotoGlobalLine)
    else:
        n = k.arg
        k.resetLabel()
        k.clearState()
        if n.isdigit():
            c.GoToLineNumber(c).go(n=int(n))
#@+node:ekr.20050929124234: *6* gotoLine
def gotoLine (self,event):

    '''Put the cursor at the n'th line of the buffer.'''

    k = self.k ; state = k.getState('goto-line')

    if state == 0:
        w = self.editWidget(event) # Sets self.w
        if not w: return
        k.setLabelBlue('Goto line: ')
        k.getArg(event,'goto-line',1,self.gotoLine)
    else:
        n = k.arg ;  w = self.w
        if n.isdigit():
            s = w.getAllText()
            i = g.convertRowColToPythonIndex(s,n,0)
            w.setInsertPoint(i)
            w.seeInsertPoint()
        k.resetLabel()
        k.clearState()
#@+node:ekr.20071114081313: *5* icons...
@

To do:

- Define standard icons in a subfolder of Icons folder?
- Tree control recomputes height of each line.
#@+node:ekr.20080108092811: *6*  Helpers
#@+node:ekr.20080108091349: *7* appendImageDictToList
def appendImageDictToList(self,aList,iconDir,path,xoffset,**kargs):

    c = self.c
    path = c.os_path_finalize_join(iconDir,path)
    relPath = g.makePathRelativeTo(path,iconDir)
    # pylint: disable=unpacking-non-sequence
    image,image_height = g.app.gui.getTreeImage(c,path)
    if not image:
        g.es('can not load image:',path)
        return xoffset
    if image_height is None:
        yoffset = 0
    else:
        yoffset = 0 # (c.frame.tree.line_height-image_height)/2
        # TNB: I suspect this is being done again in the drawing code
    newEntry = {
        'type' : 'file',
        'file' : path,
        'relPath': relPath,
        'where' : 'beforeHeadline',
        'yoffset' : yoffset, 'xoffset' : xoffset, 'xpad' : 1, # -2,
        'on' : 'VNode',
    }
    newEntry.update(kargs)  # may switch 'on' to 'VNode'
    aList.append (newEntry)
    xoffset += 2
    return xoffset
#@+node:ekr.20090701125429.6013: *7* dHash
def dHash(self, d):
    """Hash a dictionary"""
    return ''.join(['%s%s' % (str(k),str(d[k])) for k in sorted(d)])
#@+node:tbrown.20080119085249: *7* getIconList
def getIconList(self, p):
    """Return list of icons for position p, call setIconList to apply changes"""

    trace = False and not g.unitTesting
    if trace:
        if p == self.c.rootPosition(): g.trace('='*40)
        g.trace(p.h)

    fromVnode = []
    if hasattr(p.v,'unknownAttributes'):
        if trace: g.trace(p.v.u)
        fromVnode = [dict(i) for i in p.v.u.get('icons',[])]
        for i in fromVnode: i['on'] = 'VNode'

    if trace and fromVnode: g.trace('fromVnode',fromVnode,p.h)

    return fromVnode
#@+node:tbrown.20080119085249.1: *7* setIconList & helpers
def setIconList(self,p,l,setDirty=True):
    """Set list of icons for position p to l"""
    trace = False and not g.unitTesting
    current = self.getIconList(p)
    if not l and not current: return  # nothing to do
    lHash = ''.join([self.dHash(i) for i in l])
    cHash = ''.join([self.dHash(i) for i in current])
    # if trace: g.trace('lHash:',lHash)
    # if trace: g.trace('cHash:',cHash)
    if lHash == cHash:
        # no difference between original and current list of dictionaries
        return
    if trace: g.trace(l,g.callers(6))
    self._setIconListHelper(p,l,p.v,setDirty)
    self.frame.tree.updateIcon(pm, force=True)
#@+node:ekr.20090701125429.6012: *8* _setIconListHelper
def _setIconListHelper(self,p,subl,uaLoc,setDirty):
    """icon setting code common between v and t nodes

    p - postion
    subl - list of icons for the v or t node
    uaLoc - the v or t node
    """
    trace = False and not g.unitTesting
    if subl: # Update the uA.
        if not hasattr(uaLoc,'unknownAttributes'):
            uaLoc.unknownAttributes = {}
        uaLoc.unknownAttributes['icons'] = list(subl)
        # g.es((p.h,uaLoc.unknownAttributes['icons']))
        uaLoc.unknownAttributes["lineYOffset"] = 3
        uaLoc._p_changed = 1
        if setDirty:
            p.setDirty()
        if trace: g.trace('uA',uaLoc.u,uaLoc)
    else: # delete the uA.
        if hasattr(uaLoc,'unknownAttributes'):
            if 'icons' in uaLoc.unknownAttributes:
                del uaLoc.unknownAttributes['icons']
                uaLoc.unknownAttributes["lineYOffset"] = 0
                uaLoc._p_changed = 1
                if setDirty:
                    p.setDirty()
        if trace: g.trace('del uA[icons]',uaLoc)
#@+node:ekr.20071114082418: *6* deleteFirstIcon
def deleteFirstIcon (self,event=None):

    '''Delete the first icon in the selected node's icon list.'''

    c = self.c ; p = c.p

    aList = self.getIconList(p)

    if aList:
        self.setIconList(p, aList[1:])
        c.setChanged(True)
        c.redraw_after_icons_changed()
#@+node:ekr.20071114092622: *6* deleteIconByName
def deleteIconByName (self,t,name,relPath): # t not used.
    """for use by the right-click remove icon callback"""
    c = self.c ; p = c.p

    aList = self.getIconList(p)
    if not aList: return

    basePath = c.os_path_finalize_join(g.app.loadDir,"..","Icons")
    absRelPath = c.os_path_finalize_join(basePath,relPath)
    name = c.os_path_finalize(name)

    newList = []
    for d in aList:
        name2 = d.get('file')
        name2 = c.os_path_finalize(name2)
        name2rel = d.get('relPath')
        # g.trace('name',name,'\nrelPath',relPath,'\nabsRelPath',absRelPath,'\nname2',name2,'\nname2rel',name2rel)
        if not (name == name2 or absRelPath == name2 or relPath == name2rel):
            newList.append(d)

    if len(newList) != len(aList):
        self.setIconList(p, newList)       
        c.setChanged(True)
        c.redraw_after_icons_changed()
    else:
        g.trace('not found',name)
#@+node:ekr.20071114085054: *6* deleteLastIcon
def deleteLastIcon (self,event=None):

    '''Delete the first icon in the selected node's icon list.'''

    c = self.c ; p = c.p

    aList = self.getIconList(p)

    if aList:
        self.setIconList(p, aList[:-1])
        c.setChanged(True)
        c.redraw_after_icons_changed()
#@+node:ekr.20071114082418.1: *6* deleteNodeIcons
def deleteNodeIcons (self,event=None):

    '''Delete all of the selected node's icons.'''

    c = self.c ; p = c.p

    if hasattr(p.v,"unknownAttributes"):
        a = p.v.unknownAttributes
        p.v._p_changed = 1
        self.setIconList(p,[])
        a["lineYOffset"] = 0
        p.setDirty()
        c.setChanged(True)
        c.redraw_after_icons_changed()
#@+node:ekr.20071114081313.1: *6* insertIcon
def insertIcon (self,event=None):

    '''Prompt for an icon, and insert it into the node's icon list.'''

    c = self.c ; p = c.p
    iconDir = c.os_path_finalize_join(g.app.loadDir,"..","Icons")
    os.chdir(iconDir)
    paths = g.app.gui.runOpenFileDialog(
        title='Get Icons',
        filetypes=[('All files','*'),('Gif','*.gif'), ('Bitmap','*.bmp'),('Icon','*.ico'),],
        defaultextension=None,
        multiple=True)
    if not paths: return
    aList = [] ; xoffset = 2
    for path in paths:
        xoffset = self.appendImageDictToList(aList,iconDir,path,xoffset)
    aList2 = self.getIconList(p)
    aList2.extend(aList)
    self.setIconList(p, aList2)
    c.setChanged(True)
    c.redraw_after_icons_changed()
#@+node:ekr.20080108090719: *6* insertIconFromFile
def insertIconFromFile (self,path,p=None,pos=None,**kargs):

    c = self.c
    if not p: p = c.p
    iconDir = c.os_path_finalize_join(g.app.loadDir,"..","Icons")
    os.chdir(iconDir)
    aList = [] ; xoffset = 2
    xoffset = self.appendImageDictToList(aList,iconDir,path,xoffset,**kargs)
    aList2 = self.getIconList(p)
    if pos is None: pos = len(aList2)
    aList2.insert(pos,aList[0])
    self.setIconList(p, aList2)
    c.setChanged(True)
    c.redraw_after_icons_changed()
#@+node:ekr.20050920084036.74: *5* indent...
#@+node:ekr.20050920084036.76: *6* deleteIndentation
def deleteIndentation (self,event):

    '''Delete indentation in the presently line.'''

    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    line = s[i:j]
    line2 = s[i:j].lstrip()
    delta = len(line) - len(line2)
    if delta:
        self.beginCommand(undoType='delete-indentation')
        w.delete(i,j)
        w.insert(i,line2)
        ins -= delta
        w.setSelectionRange(ins,ins,insert=ins)
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.78: *6* indentRelative
def indentRelative (self,event):

    '''The indent-relative command indents at the point based on the previous
    line (actually, the last non-empty line.) It inserts whitespace at the
    point, moving point, until it is underneath an indentation point in the
    previous line.

    An indentation point is the end of a sequence of whitespace or the end of
    the line. If the point is farther right than any indentation point in the
    previous line, the whitespace before point is deleted and the first
    indentation point then applicable is used. If no indentation point is
    applicable even then whitespace equivalent to a single tab is inserted.'''

    c = self.c ; undoType = 'indent-relative' ; w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    oldSel = w.getSelectionRange()
    oldYview = w.getYScrollPosition()
    # Find the previous non-blank line
    i,j = g.getLine(s,ins)
    while 1:
        if i <= 0: return
        i,j = g.getLine(s,i-1)
        line = s[i:j]
        if line.strip(): break
    self.beginCommand(undoType=undoType)
    try:
        k = g.skip_ws(s,i)
        ws = s[i:k]
        i2,j2 = g.getLine(s,ins)
        k = g.skip_ws(s,i2)
        line = ws + s[k:j2]
        w.delete(i2,j2)
        w.insert(i2,line)
        w.setInsertPoint(i2+len(ws))
        c.frame.body.onBodyChanged(undoType,oldSel=oldSel,oldText=s,oldYview=oldYview)
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.79: *5* info...
#@+node:ekr.20050920084036.80: *6* howMany
def howMany (self,event):

    '''Print how many occurances of a regular expression are found
    in the body text of the presently selected node.'''

    k = self.k
    w = self.editWidget(event)
    if not w: return

    state = k.getState('how-many')
    if state == 0:
        k.setLabelBlue('How many: ')
        k.getArg(event,'how-many',1,self.howMany)
    else:
        k.clearState()
        s = w.getAllText()
        reg = re.compile(k.arg)
        i = reg.findall(s)
        k.setLabelGrey('%s occurances of %s' % (len(i),k.arg))
#@+node:ekr.20050920084036.81: *6* lineNumber
def lineNumber (self,event):

    '''Print the line and column number and percentage of insert point.'''

    k = self.k
    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    i = w.getInsertPoint()
    row,col = g.convertPythonIndexToRowCol(s,i)
    percent = int((i*100)/len(s))

    k.setLabelGrey(
        'char: %s row: %d col: %d pos: %d (%d%% of %d)' % (
            repr(s[i]),row,col,i,percent,len(s)))
#@+node:ekr.20050920084036.83: *6* k.viewLossage
def viewLossage (self,event):

    '''Put the Emacs-lossage in the minibuffer label.'''

    k = self.k

    g.es('lossage...')
    aList = g.app.lossage
    aList.reverse()
    for data in aList:
        ch,stroke = data
        g.es('',k.prettyPrintKey(stroke))
#@+node:ekr.20050920084036.84: *6* whatLine
def whatLine (self,event):
    '''Print the line number of the line containing the cursor.'''
    k = self.k ; w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    i = w.getInsertPoint()
    row,col = g.convertPythonIndexToRowCol(s,i)
    k.keyboardQuit()
    k.setStatusLabel("Line %s" % row)
#@+node:ekr.20050920084036.85: *5* insert & delete...
#@+node:ekr.20060417171125: *6* addSpace/TabToLines & removeSpace/TabFromLines & helper
def addSpaceToLines (self,event):
    '''Add a space to start of all lines, or all selected lines.'''
    self.addRemoveHelper(event,ch=' ',add=True,undoType='add-space-to-lines')

def addTabToLines (self,event):
    '''Add a tab to start of all lines, or all selected lines.'''
    self.addRemoveHelper(event,ch='\t',add=True,undoType='add-tab-to-lines')

def removeSpaceFromLines (self,event):
    '''Remove a space from start of all lines, or all selected lines.'''
    self.addRemoveHelper(event,ch=' ',add=False,undoType='remove-space-from-lines')

def removeTabFromLines (self,event):
    '''Remove a tab from start of all lines, or all selected lines.'''
    self.addRemoveHelper(event,ch='\t',add=False,undoType='remove-tab-from-lines')
#@+node:ekr.20060417172056: *7* addRemoveHelper
def addRemoveHelper(self,event,ch,add,undoType):

    c = self.c
    w = self.editWidget(event)
    if not w: return

    if w.hasSelection():s = w.getSelectedText()
    else:               s = w.getAllText()
    if not s: return

    # Insert or delete spaces instead of tabs when negative tab width is in effect.
    d = c.scanAllDirectives() ; width = d.get('tabwidth')
    if ch == '\t' and width < 0: ch = ' ' * abs(width)

    self.beginCommand(undoType=undoType)
    lines = g.splitLines(s)
    if add:
        result = [ch + line for line in lines]
    else:
        result = [line[len(ch):] if line.startswith(ch) else line for line in lines]
    result = ''.join(result)
    # g.trace('add',add,'hasSelection',w.hasSelection(),'result',repr(result))
    if w.hasSelection():
        i,j = w.getSelectionRange()
        w.delete(i,j)
        w.insert(i,result)
        w.setSelectionRange(i,i+len(result))
    else:
        w.setAllText(result)
        w.setSelectionRange(0,len(s))
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20051026092433.1: *6* backwardDeleteCharacter
def backwardDeleteCharacter (self,event=None):

    '''Delete the character to the left of the cursor.'''

    c = self.c ; p = c.p
    w = self.editWidget(event)
    if not w: return

    wname = c.widget_name(w)
    ins = w.getInsertPoint()
    i,j = w.getSelectionRange()
    # g.trace(wname,i,j,ins)

    if wname.startswith('body'):
        self.beginCommand()
        try:
            d = c.scanAllDirectives(p)
            tab_width = d.get("tabwidth",c.tab_width)
            changed = True
            if i != j:
                w.delete(i,j)
                w.setSelectionRange(i,i,insert=i)
            elif i == 0:
                changed = False
            elif tab_width > 0:
                w.delete(ins-1)
                w.setSelectionRange(ins-1,ins-1,insert=ins-1)
            else:
                << backspace with negative tab_width >>
        finally:
            self.endCommand(changed=changed,setLabel=False)
                # Necessary to make text changes stick.
    else:
        # No undo in this widget.
        # Make sure we actually delete something if we can.
        s = w.getAllText()
        if i != j:
            j = max(i,min(j,len(s)))
            w.delete(i,j)
            w.setSelectionRange(i,i,insert=i)
        elif ins != 0:
            # Do nothing at the start of the headline.
            w.delete(ins-1)
            ins = ins-1
            w.setSelectionRange(ins,ins,insert=ins)
#@+node:ekr.20051026092746: *7* << backspace with negative tab_width >>
s = prev = w.getAllText()
ins = w.getInsertPoint()
i,j = g.getLine(s,ins)
s = prev = s[i:ins]
n = len(prev)
abs_width = abs(tab_width)

# Delete up to this many spaces.
n2 = (n % abs_width) or abs_width
n2 = min(n,n2) ; count = 0

while n2 > 0:
    n2 -= 1
    ch = prev[n-count-1]
    if ch != ' ': break
    else: count += 1

# Make sure we actually delete something.
i = ins-(max(1,count))
w.delete(i,ins)
w.setSelectionRange(i,i,insert=i)
#@+node:ekr.20070325094935: *6* cleanAllLines
def cleanAllLines (self,event):
    '''Clean all lines in the selected tree.'''
    c = self.c
    u = c.undoer
    w = c.frame.body.wrapper
    if not w: return
    tag = 'clean-all-lines'
    u.beforeChangeGroup(c.p,tag)
    n = 0
    for p in c.p.self_and_subtree():
        lines = []
        for line in g.splitLines(p.b):
            if line.rstrip():
                lines.append(line.rstrip())
            if line.endswith('\n'):
                lines.append('\n')
        s2 = ''.join(lines)
        if s2 != p.b:
            print(p.h)
            bunch = u.beforeChangeNodeContents(p)
            p.b = s2
            p.v.setDirty()
            n += 1
            u.afterChangeNodeContents(p,tag,bunch)
    u.afterChangeGroup(c.p,tag)
    c.redraw_after_icons_changed()
    g.es('cleaned %s nodes' % n)
#@+node:ekr.20060415112257: *6* cleanLines
def cleanLines (self,event):

    '''Removes trailing whitespace from all lines, preserving newlines.
    '''

    w = self.editWidget(event)
    if not w: return
    if w.hasSelection():
        s = w.getSelectedText()
    else:
        s = w.getAllText()
    lines = []
    for line in g.splitlines(s):
        if line.rstrip():
            lines.append(line.rstrip())
        if line.endswith('\n'):
            lines.append('\n')
    result = ''.join(lines)
    if s != result:
        self.beginCommand(undoType='clean-lines')
        if w.hasSelection():
            i,j = w.getSelectionRange()
            w.delete(i,j)
            w.insert(i,result)
            w.setSelectionRange(i,j+len(result))
        else:
            i = w.getInsertPoint()
            w.delete(0,'end')
            w.insert(0,result)
            w.setInsertPoint(i)
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20060414085834: *6* clearSelectedText
def clearSelectedText (self,event):

    '''Delete the selected text.'''

    w = self.editWidget(event)
    if not w: return
    i,j = w.getSelectionRange()
    if i == j: return
    self.beginCommand(undoType='clear-selected-text')
    w.delete(i,j)
    w.setInsertPoint(i)
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20100817125519.5833: *6* delete-word & backward-delete-word
def deleteWord(self,event=None):
    '''Delete the word at the cursor.'''
    self.deleteWordHelper(event,forward=True)

def backwardDeleteWord(self,event=None):
    '''Delete the word in front of the cursor.'''
    self.deleteWordHelper(event,forward=False)

# Patch by NH2.
def deleteWordSmart(self,event=None):
    '''Delete the word at the cursor, treating whitespace
    and symbols smartly.'''
    self.deleteWordHelper(event,forward=True,smart=True)

def backwardDeleteWordSmart(self,event=None):
    '''Delete the word in front of the cursor, treating whitespace
    and symbols smartly.'''
    self.deleteWordHelper(event,forward=False,smart=True)

def deleteWordHelper(self,event,forward,smart=False):
    c = self.c ; w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType="delete-word")
    if w.hasSelection():
        from_pos,to_pos = w.getSelectionRange()
    else:
        from_pos = w.getInsertPoint()
        self.moveWordHelper(event,extend=False,forward=forward,smart=smart)
        to_pos = w.getInsertPoint()

    # For Tk GUI, make sure to_pos > from_pos
    if from_pos > to_pos:
        from_pos,to_pos = to_pos,from_pos

    w.delete(from_pos,to_pos)
    c.frame.body.forceFullRecolor()
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.87: *6* deleteNextChar
def deleteNextChar (self,event):

    '''Delete the character to the right of the cursor.'''

    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    i,j = w.getSelectionRange()
    self.beginCommand(undoType='delete-char')
    changed = True
    if i != j:
        w.delete(i,j)
        w.setInsertPoint(i)
    elif j < len(s):
        w.delete(i)
        w.setInsertPoint(i)
    else:
        changed = False
    self.endCommand(changed=changed,setLabel=False)
#@+node:ekr.20050920084036.135: *6* deleteSpaces
def deleteSpaces (self,event,insertspace=False):

    '''Delete all whitespace surrounding the cursor.'''

    w = self.editWidget(event)
    if not w: return
    undoType = 'insert-space' if insertspace else 'delete-spaces'
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    w1 = ins-1
    while w1 >= i and s[w1].isspace():
        w1 -= 1
    w1 += 1
    w2 = ins
    while w2 <= j and s[w2].isspace():
        w2 += 1
    spaces = s[w1:w2]
    if spaces:
        self.beginCommand(undoType=undoType)
        if insertspace: s = s[:w1] + ' ' + s[w2:]
        else:           s = s[:w1] + s[w2:]
        w.setAllText(s)
        w.setInsertPoint(w1)
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20110528103005.18328: *6* insertHardTab
def insertHardTab(self,event):

    '''Insert one hard tab.'''

    c = self.c
    w = self.editWidget(event)
    if not w: return
    assert g.isTextWrapper(w)
    name = c.widget_name(w)
    if name.startswith('head'): return
    ins = w.getInsertPoint()
    self.beginCommand(undoType='insert-hard-tab')
    w.insert(ins,'\t')
    ins += 1
    w.setSelectionRange(ins,ins,insert=ins)
    self.endCommand()
#@+node:ekr.20050920084036.138: *6* insertNewLine
def insertNewLine (self,event):

    '''Insert a newline at the cursor.'''

    c = self.c ; k = c.k ; w = self.editWidget(event)
    if not w: return

    assert g.isTextWrapper(w)
    name = c.widget_name(w)
    if name.startswith('head'): return

    oldSel = w.getSelectionRange()
    # g.trace('oldSel',oldSel)

    self.beginCommand(undoType='newline')

    # New in Leo 4.5: use the same logic as in selfInsertCommand.
    self.insertNewlineHelper(w=w,oldSel=oldSel,undoType=None)
    k.setInputState('insert')
    k.showStateAndMode()

    self.endCommand()

insertNewline = insertNewLine
#@+node:ekr.20050920084036.86: *6* insertNewLineAndTab
def insertNewLineAndTab (self,event):

    '''Insert a newline and tab at the cursor.'''

    c = self.c ; k = c.k
    w = self.editWidget(event) ; p = c.p
    if not w: return

    assert g.isTextWrapper(w),w
    name = c.widget_name(w)
    if name.startswith('head'): return

    self.beginCommand(undoType='insert-newline-and-indent')

    # New in Leo 4.5: use the same logic as in selfInsertCommand.
    oldSel = w.getSelectionRange()
    self.insertNewlineHelper(w=w,oldSel=oldSel,undoType=None)
    self.updateTab(p,w,smartTab=False)
    k.setInputState('insert')
    k.showStateAndMode()

    self.endCommand(changed=True,setLabel=False)
#@+node:ekr.20050920084036.139: *6* insertParentheses
def insertParentheses (self,event):

    '''Insert () at the cursor.'''

    w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType='insert-parenthesis')

    i = w.getInsertPoint()
    w.insert(i,'()')
    w.setInsertPoint(i+1)

    self.endCommand(changed=True,setLabel=False)
#@+node:ekr.20110528103005.18329: *6* insertSoftTab
def insertSoftTab (self,event):

    '''Insert spaces equivalent to one tab.'''

    c = self.c ; p = c.p
    w = self.editWidget(event) 
    if not w: return

    assert g.isTextWrapper(w)
    name = c.widget_name(w)
    if name.startswith('head'): return

    d = c.scanAllDirectives(p)
    n = abs(d.get("tabwidth",c.tab_width))
    ins = w.getInsertPoint()

    self.beginCommand(undoType='insert-soft-tab')

    w.insert(ins,' ' * n)
    ins += n
    w.setSelectionRange(ins,ins,insert=ins)

    self.endCommand()

#@+node:ekr.20050920084036.141: *6* removeBlankLines
def removeBlankLines (self,event):

    '''The remove-blank-lines command removes lines containing nothing but
    whitespace. If there is a text selection, only lines within the selected
    text are affected; otherwise all blank lines in the selected node are
    affected.'''

    c = self.c
    head,lines,tail,oldSel,oldYview = c.getBodyLines()

    changed = False ; result = []
    for line in lines:
        if line.strip():
            result.append(line)
        else:
            changed = True
    result = ''.join(result)

    if changed:
        oldSel = None ; undoType = 'remove-blank-lines'
        c.updateBodyPane(head,result,tail,undoType,oldSel,oldYview)
#@+node:ekr.20110530082209.18248: *6* replaceCurrentCharacter
def replaceCurrentCharacter (self,event):

    '''Replace the current character with the next character typed.'''

    k = self.k ; tag = 'replace-current-character'
    state = k.getState(tag)

    if state == 0:
        w = self.editWidget(event) # sets self.w
        if w:
            k.setLabelBlue('Replace Character: ')
            k.getArg(event,tag,1,self.replaceCurrentCharacter)
    else:
        w = self.w
        ch = k.arg
        if ch:
            i,j = w.getSelectionRange()
            if i > j: i,j = j,i
            # Use raw insert/delete to retain the coloring.
            if i == j:
                i = max(0,i-1)
                w.delete(i)
            else:
                w.delete(i,j)
            w.insert(i,ch)
            w.setInsertPoint(i+1)
        k.clearState()
        k.resetLabel()
        k.showStateAndMode()
#@+node:ekr.20051125080855: *6* selfInsertCommand, helpers
def selfInsertCommand(self,event,action='insert'):

    '''Insert a character in the body pane.
    This is the default binding for all keys in the body pane.'''

    trace = False and not g.unitTesting
    c,k = self.c,self.k
    verbose = True
    w = self.editWidget(event)
    if not w: return # (for Tk) 'break'
    << set local vars >>
    assert g.isStrokeOrNone(stroke)

    if trace: g.trace('ch',repr(ch),'stroke',stroke)
    if g.doHook("bodykey1",c=c,p=p,v=p,ch=ch,oldSel=oldSel,undoType=undoType):
        return # (for Tk) "break" # The hook claims to have handled the event.
    if ch == '\t':
        self.updateTab(p,w)
    elif ch == '\b':
        # This is correct: we only come here if there no bindngs for this key. 
        self.backwardDeleteCharacter(event)
    elif ch in ('\r','\n'):
        ch = '\n'
        self.insertNewlineHelper(w,oldSel,undoType)
    elif inBrackets and self.autocompleteBrackets:
        self.updateAutomatchBracket(p,w,ch,oldSel)
    elif ch: # Null chars must not delete the selection.
        isPlain = stroke.find('Alt') == -1 and stroke.find('Ctrl') == -1
        i,j = oldSel
        if i > j: i,j = j,i
        # Use raw insert/delete to retain the coloring.
        if i != j:                  w.delete(i,j)
        elif action == 'overwrite': w.delete(i)
        if isPlain: # 2013/10/07: call insertKeyEvent for non-plain characters.
            w.insert(i,ch)
            w.setInsertPoint(i+1)
        else:
            g.app.gui.insertKeyEvent(event,i)
        if inBrackets and self.flashMatchingBrackets:
            self.flashMatchingBracketsHelper(w,i,ch)               
    else:
        return # (for Tk) 'break' # This method *always* returns 'break'

    # Set the column for up and down keys.
    spot = w.getInsertPoint()
    c.editCommands.setMoveCol(w,spot)

    # Update the text and handle undo.
    newText = w.getAllText()
    changed = newText != oldText
    if trace and verbose:
        g.trace('ch',repr(ch),'changed',changed,'newText',repr(newText[-10:]))
    if changed:
        # g.trace('ins',w.getInsertPoint())
        c.frame.body.onBodyChanged(undoType=undoType,
            oldSel=oldSel,oldText=oldText,oldYview=None)

    g.doHook("bodykey2",c=c,p=p,v=p,ch=ch,oldSel=oldSel,undoType=undoType)
    return # (for Tk) 'break'
#@+node:ekr.20061103114242: *7* << set local vars >>
p = c.p

stroke = event and event.stroke or None
ch = event and event.char or ''

if ch == 'Return':
    ch = '\n' # This fixes the MacOS return bug.
if ch == 'Tab':
    ch = '\t'

name = c.widget_name(w)
oldSel =  name.startswith('body') and w.getSelectionRange() or (None,None)
oldText = name.startswith('body') and p.b or ''
undoType = 'Typing'
brackets = self.openBracketsList + self.closeBracketsList
inBrackets = ch and g.toUnicode(ch) in brackets
# if trace: g.trace(name,repr(ch),ch and ch in brackets)
#@+node:ekr.20090213065933.14: *7* doPlainTab
def doPlainTab(self,s,i,tab_width,w):

    '''Insert spaces equivalent to one tab.'''

    start,end = g.getLine(s,i)
    s2 = s[start:i]
    width = g.computeWidth(s2,tab_width)

    if tab_width > 0:
        w.insert(i,'\t')
        ins = i+1
    else:
        n = abs(tab_width) - (width % abs(tab_width))
        w.insert(i,' ' * n)
        ins = i+n

    w.setSelectionRange(ins,ins,insert=ins)
#@+node:ekr.20060627091557: *7* flashCharacter (leoEditCommands)
def flashCharacter(self,w,i):

    bg      = self.bracketsFlashBg or 'DodgerBlue1'
    fg      = self.bracketsFlashFg or 'white'
    flashes = self.bracketsFlashCount or 3
    delay   = self.bracketsFlashDelay or 75

    w.flashCharacter(i,bg,fg,flashes,delay)
#@+node:ekr.20060627083506: *7* flashMatchingBracketsHelper
def flashMatchingBracketsHelper (self,w,i,ch):

    d = {}
    if ch in self.openBracketsList:
        for z in range(len(self.openBracketsList)):
            d [self.openBracketsList[z]] = self.closeBracketsList[z]
        reverse = False # Search forward
    else:
        for z in range(len(self.openBracketsList)):
            d [self.closeBracketsList[z]] = self.openBracketsList[z]
        reverse = True # Search backward

    delim2 = d.get(ch)

    s = w.getAllText()
    j = g.skip_matching_python_delims(s,i,ch,delim2,reverse=reverse)
    if j != -1:
        self.flashCharacter(w,j)
#@+node:ekr.20060804095512: *7* initBracketMatcher
def initBracketMatcher (self,c):

    if len(self.openBracketsList) != len(self.closeBracketsList):

        g.es_print('bad open/close_flash_brackets setting: using defaults')
        self.openBracketsList  = '([{'
        self.closeBracketsList = ')]}'

    # g.trace('self.openBrackets',openBrackets)
    # g.trace('self.closeBrackets',closeBrackets)
#@+node:ekr.20051026171121: *7* insertNewlineHelper
def insertNewlineHelper (self,w,oldSel,undoType):

    trace = False and not g.unitTesting
    c = self.c ; p = c.p
    i,j = oldSel ; ch = '\n'
    if trace:
        s = w.widget.toPlainText()
        g.trace(i,j,len(s),w)

    if i != j:
        # No auto-indent if there is selected text.
        w.delete(i,j)
        w.insert(i,ch)
        w.setInsertPoint(i+1)
    else:
        w.insert(i,ch)
        w.setInsertPoint(i+1)

        if (c.autoindent_in_nocolor or 
            (c.frame.body.colorizer.useSyntaxColoring(p) and
            undoType != "Change")
        ):
            # No auto-indent if in @nocolor mode or after a Change command.
            self.updateAutoIndent(p,w)

    w.seeInsertPoint()
#@+node:ekr.20051026171121.1: *7* updateAutoIndent (leoEditCommands)
def updateAutoIndent (self,p,w):

    trace = False and not g.unitTesting
    c = self.c
    d = c.scanAllDirectives(p)
    tab_width = d.get("tabwidth",c.tab_width)
    # Get the previous line.
    s = w.getAllText()
    ins = w.getInsertPoint()
    i = g.skip_to_start_of_line(s,ins)
    i,j = g.getLine(s,i-1)
    s = s[i:j-1]

    # Add the leading whitespace to the present line.
    junk, width = g.skip_leading_ws_with_indent(s,0,tab_width)
    # g.trace('width',width,'tab_width',tab_width)

    if s and s [-1] == ':':
        # For Python: increase auto-indent after colons.
        if g.findLanguageDirectives(c,p) == 'python':
            width += abs(tab_width)

    if self.smartAutoIndent:
        # Determine if prev line has unclosed parens/brackets/braces
        bracketWidths = [width] ; tabex = 0
        for i in range(0,len(s)):
            if s [i] == '\t':
                tabex += tab_width-1
            if s [i] in '([{':
                bracketWidths.append(i+tabex+1)
            elif s [i] in '}])' and len(bracketWidths) > 1:
                bracketWidths.pop()
        width = bracketWidths.pop()

    ws = g.computeLeadingWhitespace(width,tab_width)
    if ws:
        if trace: g.trace('width: %s, tab_width: %s, ws: %s' % (
            width,tab_width,repr(ws)))
        i = w.getInsertPoint()
        w.insert(i,ws)
        w.setInsertPoint(i+len(ws))
        w.seeInsertPoint()
            # 2011/10/02: Fix cursor-movement bug.
#@+node:ekr.20051027172949: *7* updateAutomatchBracket
def updateAutomatchBracket (self,p,w,ch,oldSel):

    # assert ch in ('(',')','[',']','{','}')

    c = self.c ; d = c.scanAllDirectives(p)
    i,j = oldSel
    language = d.get('language')
    s = w.getAllText()

    if ch in ('(','[','{',):
        automatch = language not in ('plain',)
        if automatch:
            ch = ch + {'(':')','[':']','{':'}'}.get(ch)
        if i != j: w.delete(i,j)
        w.insert(i,ch)
        if automatch:
            ins = w.getInsertPoint()
            w.setInsertPoint(ins-1)
    else:
        ins = w.getInsertPoint()
        ch2 = ins<len(s) and s[ins] or ''
        if ch2 in (')',']','}'):
            ins = w.getInsertPoint()
            w.setInsertPoint(ins+1)
        else:
            if i != j: w.delete(i,j)
            w.insert(i,ch)
            w.setInsertPoint(i+1)
#@+node:ekr.20051026092433: *7* updateTab
def updateTab (self,p,w,smartTab=True):

    trace = False and not g.unitTesting
    c = self.c

    # g.trace('tab_width',tab_width)
    i,j = w.getSelectionRange()
        # Returns insert point if no selection, with i <= j.

    if i != j:
        # w.delete(i,j)
        c.indentBody()
    else:
        d = c.scanAllDirectives(p)
        tab_width = d.get("tabwidth",c.tab_width)
        if trace: g.trace(tab_width)

        # Get the preceeding characters.
        s = w.getAllText()
        start,end = g.getLine(s,i)
        after = s[i:end]
        if after.endswith('\n'): after = after[:-1]

        # Only do smart tab at the start of a blank line.
        doSmartTab = (smartTab and c.smart_tab and i == start)
            # Truly at the start of the line.
            # and not after # Nothing *at all* after the cursor.

        if trace:
            g.trace('smartTab',doSmartTab,'tab_width',tab_width)
                # 'i %s start %s after %s' % (i,start,repr(after)))

        if doSmartTab:
            self.updateAutoIndent(p,w)
            # Add a tab if otherwise nothing would happen.
            if s == w.getAllText():
                self.doPlainTab(s,i,tab_width,w)
        else:
            self.doPlainTab(s,i,tab_width,w)
#@+node:ekr.20140816053742.18402: *5* insertFileName
def insertFileName(self,event=None):
    '''
    Prompt for a file name, then insert it at the cursor position.
    This operation is undoable if done in the body pane.
    '''
    c = self.c
    w = self.editWidget(event)
    if w:
        def callback(arg,w=w):
            i = w.getInsertPoint()
            w.insert(i,arg)
            if g.app.gui.widget_name(w) == 'body':
                c.frame.body.onBodyChanged(undoType='Typing')
        c.k.getFileName(event,callback=callback)
#@+node:ekr.20031218072017.3983: *5* insertHeadlineTime
def insertHeadlineTime (self,event=None):
    '''Insert a date/time stamp in the headline of the selected node.'''
    frame = self ; c = frame.c ; p = c.p
    if g.app.batchMode:
        c.notValidInBatchMode("Insert Headline Time")
        return
    w = self.editWidget(event)
    if w:
        # Fix bug https://bugs.launchpad.net/leo-editor/+bug/1185933
        # insert-headline-time should insert at cursor.
        # Note: The command must be bound to a key for this to work.
        ins = w.getInsertPoint()
        s = c.getTime(body=False)
        w.insert(ins,s)
    else:
        c.endEditing()
        time = c.getTime(body=False)
        s = p.h.rstrip()
        if s:
            p.h = ' '.join([s, time])
        else:
            p.h = time
        
        c.redrawAndEdit(p,selectAll=True)
#@+node:ekr.20050920084036.88: *5* line...
#@+node:ekr.20050920084036.90: *6* flushLines
def flushLines (self,event):

    '''Delete each line that contains a match for regexp, operating on the text after point.

    In Transient Mark mode, if the region is active, the command operates on the region instead.'''

    k = self.k ; state = k.getState('flush-lines')

    if state == 0:
        k.setLabelBlue('Flush lines regexp: ')
        k.getArg(event,'flush-lines',1,self.flushLines)
    else:
        k.clearState()
        k.resetLabel()
        self.linesHelper(event,k.arg,'flush')
        k.commandName = 'flush-lines %s' % k.arg
#@+node:ekr.20051002095724: *6* keepLines
def keepLines (self,event):

    '''Delete each line that does not contain a match for regexp, operating on the text after point.

    In Transient Mark mode, if the region is active, the command operates on the region instead.'''

    k = self.k ; state = k.getState('keep-lines')

    if state == 0:
        k.setLabelBlue('Keep lines regexp: ')
        k.getArg(event,'keep-lines',1,self.keepLines)
    else:
        k.clearState()
        k.resetLabel()
        self.linesHelper(event,k.arg,'keep')
        k.commandName = 'keep-lines %s' % k.arg
#@+node:ekr.20050920084036.92: *6* linesHelper
def linesHelper (self,event,pattern,which):

    w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType=which+'-lines')
    if w.hasSelection():
        i,end = w.getSelectionRange()
    else:
        i = w.getInsertPoint()
        end = 'end'
    txt = w.get(i,end)
    tlines = txt.splitlines(True)
    if which == 'flush':    keeplines = list(tlines)
    else:                   keeplines = []

    try:
        regex = re.compile(pattern)
        for n, z in enumerate(tlines):
            f = regex.findall(z)
            if which == 'flush' and f:
                keeplines [n] = None
            elif f:
                keeplines.append(z)
    except Exception:
        return
    if which == 'flush':
        keeplines = [x for x in keeplines if x != None]
    w.delete(i,end)
    w.insert(i,''.join(keeplines))
    w.setInsertPoint(i)
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.77: *6* splitLine
def splitLine (self,event):

    '''Split a line at the cursor position.'''

    w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType='split-line')

    s = w.getAllText()
    ins = w.getInsertPoint()
    w.setAllText(s[:ins] + '\n' + s[ins:])
    w.setInsertPoint(ins+1)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050929114218: *5* move cursor... (leoEditCommands)
#@+node:ekr.20051218170358: *6*  general helpers
#@+node:ekr.20060113130510: *7* extendHelper
def extendHelper (self,w,extend,spot,upOrDown=False):
    '''Handle the details of extending the selection.
    This method is called for all cursor moves.

    extend: Clear the selection unless this is True.
    spot:   The *new* insert point.
    '''

    trace = False and not g.unitTesting
    verbose = False
    c = self.c ; p = c.p
    extend = extend or self.extendMode

    ins = w.getInsertPoint()
    i,j = w.getSelectionRange()
    if trace: g.trace(
        'extend',extend,'ins',ins,'sel=',i,j,
        'spot=',spot,'moveSpot',self.moveSpot)

    # Reset the move spot if needed.
    if self.moveSpot is None or p.v != self.moveSpotNode:
        if trace: g.trace('no spot')
        self.setMoveCol(w,ins if extend else spot) # sets self.moveSpot.
    elif extend:
        # 2011/05/20: Fix bug 622819
        # Ctrl-Shift movement is incorrect when there is an unexpected selection.
        if i == j:
            if trace: g.trace('extend and no sel')
            self.setMoveCol(w,ins) # sets self.moveSpot.
        elif self.moveSpot in (i,j) and self.moveSpot != ins:
            if trace and verbose: g.trace('extend and movespot matches')
            # The bug fix, part 1.
        else:
            # The bug fix, part 2.
            # Set the moveCol to the *not* insert point.
            if ins == i: k = j
            elif ins == j: k = i
            else: k = ins
            if trace: g.trace('extend and unexpected spot',k)
            self.setMoveCol(w,k) # sets self.moveSpot.
    else:
        if upOrDown:
            s = w.getAllText()
            i2,j2 = g.getLine(s,spot)
            line = s[i2:j2]
            row,col = g.convertPythonIndexToRowCol(s,spot)
            if True: # was j2 < len(s)-1:
                n = min(self.moveCol,max(0,len(line)-1))
            else:
                n = min(self.moveCol,max(0,len(line))) # A tricky boundary.
            # g.trace('using moveCol',self.moveCol,'line',repr(line),'n',n)
            spot = g.convertRowColToPythonIndex(s,row,n)
        else:  # Plain move forward or back.
            # g.trace('plain forward/back move')
            self.setMoveCol(w,spot) # sets self.moveSpot.

    if extend:
        if trace: g.trace('range',spot,self.moveSpot)
        if spot < self.moveSpot:
            w.setSelectionRange(spot,self.moveSpot,insert=spot)
        else:
            w.setSelectionRange(self.moveSpot,spot,insert=spot)
    else:
        if trace: g.trace('insert point',spot)
        w.setSelectionRange(spot,spot,insert=spot)

    w.seeInsertPoint()
    c.frame.updateStatusLine()
#@+node:ekr.20051218122116: *7* moveToHelper (leoEditCommands)
def moveToHelper (self,event,spot,extend):

    '''Common helper method for commands the move the cursor
    in a way that can be described by a Tk Text expression.'''

    c = self.c ; k = c.k ; w = self.editWidget(event)
    if not w: return

    c.widgetWantsFocusNow(w)

    # Put the request in the proper range.
    if c.widget_name(w).startswith('mini'):
        i,j = k.getEditableTextRange()
        if   spot < i: spot = i
        elif spot > j: spot = j

    self.extendHelper(w,extend,spot,upOrDown=False)
#@+node:ekr.20060209095101: *7* setMoveCol
def setMoveCol (self,w,spot):

    '''Set the column to which an up or down arrow will attempt to move.'''

    c = self.c ; p = c.p

    i,row,col = w.toPythonIndexRowCol(spot)

    self.moveSpot = i
    self.moveCol = col
    self.moveSpotNode = p.v

    # g.trace('moveSpot',i)
#@+node:ekr.20081123102100.1: *6* backToHome/ExtendSelection
def backToHome (self,event,extend=False):
    '''
    Smart home:
    Position the point at the first non-blank character on the line,
    or the start of the line if already there.
    '''
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    if s:
        i,j = g.getLine(s,ins)
        i1 = i
        while i < j and s[i] in (' \t'):
            i += 1
        if i == ins:
            i = i1
        self.moveToHelper(event,i,extend=extend)
        
def backToHomeExtendSelection (self,event):
    self.backToHome(event,extend=True)
#@+node:ekr.20050920084036.75: *6* backToIndentation
def backToIndentation (self,event):

    '''Position the point at the first non-blank character on the line.'''

    w = self.editWidget(event)
    if not w: return

    # None of the other cursor move commands are undoable.
    # self.beginCommand(undoType='back-to-indentation')

    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    while i < j and s[i] in (' \t'):
        i += 1

    self.moveToHelper(event,i,extend=False)

    # self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20051218141237: *6* between lines & helper
def nextLine (self,event):
    '''Move the cursor down, extending the selection if in extend mode.'''
    self.moveUpOrDownHelper(event,'down',extend=False)

def nextLineExtendSelection (self,event):
    '''Extend the selection by moving the cursor down.'''
    self.moveUpOrDownHelper(event,'down',extend=True)

def prevLine (self,event):
    '''Move the cursor up, extending the selection if in extend mode.'''
    self.moveUpOrDownHelper(event,'up',extend=False)

def prevLineExtendSelection (self,event):
    '''Extend the selection by moving the cursor up.'''
    self.moveUpOrDownHelper(event,'up',extend=True)
#@+node:ekr.20060113105246.1: *7* moveUpOrDownHelper
def moveUpOrDownHelper (self,event,direction,extend):

    trace = False and not g.unitTesting
    w = self.editWidget(event)
    if not w: return

    ins = w.getInsertPoint()
    s = w.getAllText()
    w.seeInsertPoint()

    if hasattr(w,'leoMoveCursorHelper'):
        extend = extend or self.extendMode
        w.leoMoveCursorHelper(kind=direction,extend=extend)
    else:
        # Find the start of the next/prev line.
        row,col = g.convertPythonIndexToRowCol(s,ins)
        if trace:
            gui_ins = w.toPythonIndex(ins)
            bbox = w.bbox(gui_ins)
            if bbox:
                x,y,width,height = bbox
                # bbox: x,y,width,height;  dlineinfo: x,y,width,height,offset
                g.trace('gui_ins',gui_ins,'dlineinfo',w.dlineinfo(gui_ins),'bbox',bbox)
                g.trace('ins',ins,'row',row,'col',col,
                    'event.x',event and event.x,'event.y',event and event.y)
                g.trace('subtracting line height',w.index('@%s,%s' % (x,y-height)))
                g.trace('adding      line height',w.index('@%s,%s' % (x,y+height)))
        i,j = g.getLine(s,ins)
        if direction == 'down':
            i2,j2 = g.getLine(s,j)
        else:
            i2,j2 = g.getLine(s,i-1)

        # The spot is the start of the line plus the column index.
        n = max(0,j2-i2-1) # The length of the new line.
        col2 = min(col,n)
        spot = i2 + col2
        if trace: g.trace('spot',spot,'n',n,'col',col,'line',repr(s[i2:j2]))

        self.extendHelper(w,extend,spot,upOrDown=True)
#@+node:ekr.20050920084036.148: *6* buffers & helper
def beginningOfBuffer (self,event):
    '''Move the cursor to the start of the body text.'''
    self.moveToBufferHelper(event,'home',extend=False)

def beginningOfBufferExtendSelection (self,event):
    '''Extend the text selection by moving the cursor to the start of the body text.'''
    self.moveToBufferHelper(event,'home',extend=True)

def endOfBuffer (self,event):
    '''Move the cursor to the end of the body text.'''
    self.moveToBufferHelper(event,'end',extend=False)

def endOfBufferExtendSelection (self,event):
    '''Extend the text selection by moving the cursor to the end of the body text.'''
    self.moveToBufferHelper(event,'end',extend=True)
#@+node:ekr.20100109094541.6227: *7* moveToBufferHelper
def moveToBufferHelper (self,event,spot,extend):

    w = self.editWidget(event)
    if not w: return

    if hasattr(w,'leoMoveCursorHelper'):
        extend = extend or self.extendMode
        w.leoMoveCursorHelper(kind=spot,extend=extend)
    else:
        if spot == 'home':
            self.moveToHelper(event,0,extend=extend)
        elif spot == 'end':
            s = w.getAllText()
            self.moveToHelper(event,len(s),extend=extend)
        else:
            g.trace('can not happen: bad spot',spot)
#@+node:ekr.20051213080533: *6* characters & helper
def backCharacter (self,event):
    '''Move the cursor back one character, extending the selection if in extend mode.'''
    self.moveToCharacterHelper(event,'left',extend=False)

def backCharacterExtendSelection (self,event):
    '''Extend the selection by moving the cursor back one character.'''
    self.moveToCharacterHelper(event,'left',extend=True)

def forwardCharacter (self,event):
    '''Move the cursor forward one character, extending the selection if in extend mode.'''
    self.moveToCharacterHelper(event,'right',extend=False)

def forwardCharacterExtendSelection (self,event):
    '''Extend the selection by moving the cursor forward one character.'''
    self.moveToCharacterHelper(event,'right',extend=True)
#@+node:ekr.20100109094541.6228: *7* moveToCharacterHelper
def moveToCharacterHelper (self,event,spot,extend):

    w = self.editWidget(event)
    if not w: return

    if hasattr(w,'leoMoveCursorHelper'):
        extend = extend or self.extendMode
        w.leoMoveCursorHelper(kind=spot,extend=extend)
    else:
        i = w.getInsertPoint()
        if spot == 'left':
            i=max(0,i-1)
            self.moveToHelper(event,i,extend=extend)
        elif spot == 'right':
            i = min(i+1,len(w.getAllText()))
            self.moveToHelper(event,i,extend=extend)
        else:
            g.trace('can not happen: bad spot: %s' % spot)
#@+node:ekr.20051218174113: *6* clear/set/ToggleExtendMode
def clearExtendMode (self,event):
    '''Turn off extend mode: cursor movement commands do not extend the selection.'''
    self.extendModeHelper(event,False)

def setExtendMode (self,event):
    '''Turn on extend mode: cursor movement commands do extend the selection.'''
    self.extendModeHelper(event,True)

def toggleExtendMode (self,event):
    '''Toggle extend mode, i.e., toggle whether cursor movement commands extend the selections.'''
    self.extendModeHelper(event,not self.extendMode)

def extendModeHelper (self,event,val):

    c = self.c
    w = self.editWidget(event)
    if not w: return

    self.extendMode = val
    if not g.unitTesting:
        # g.red('extend mode','on' if val else 'off'))
        c.k.showStateAndMode()
    c.widgetWantsFocusNow(w)
#@+node:ekr.20050920084036.136: *6* exchangePointMark
def exchangePointMark (self,event):

    '''Exchange the point (insert point) with the mark (the other end of the selected text).'''

    c = self.c
    w = self.editWidget(event)
    if not w: return

    if hasattr(w,'leoMoveCursorHelper'):
        w.leoMoveCursorHelper(kind='exchange',extend=False)
    else:
        c.widgetWantsFocusNow(w)
        i,j = w.getSelectionRange(sort=False)
        if i == j: return
        ins = w.getInsertPoint()
        ins = j if ins==i else i
        w.setInsertPoint(ins)
        w.setSelectionRange(i,j,insert=None)
#@+node:ekr.20061007082956: *6* extend-to-line
def extendToLine (self,event):

    '''Select the line at the cursor.'''

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText() ; n = len(s)
    i = w.getInsertPoint()

    while 0 <= i < n and not s[i] == '\n':
        i -= 1
    i += 1 ; i1 = i
    while 0 <= i < n and not s[i] == '\n':
        i += 1

    w.setSelectionRange(i1,i)
#@+node:ekr.20061007214835.4: *6* extend-to-sentence
def extendToSentence (self,event):

    '''Select the line at the cursor.'''

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText() ; n = len(s)
    i = w.getInsertPoint()

    i2 = 1 + s.find('.',i)
    if i2 == -1: i2 = n
    i1 = 1 + s.rfind('.',0,i2-1)

    w.setSelectionRange(i1,i2)
#@+node:ekr.20060116074839.2: *6* extend-to-word
def extendToWord (self,event,direction='forward',select=True):

    '''Compute the word at the cursor. Select it if select arg is True.'''

    w = self.editWidget(event)
    if not w:
        return 0,0
    s = w.getAllText() ; n = len(s)
    i = w.getInsertPoint()
    if direction == 'forward':
        while i < n and not g.isWordChar(s[i]):
            i += 1
    else:
        while 0 <= i < n and not g.isWordChar(s[i]):
            i -= 1
    while 0 <= i < n and g.isWordChar(s[i]):
        i -= 1
    i += 1
    i1 = i
    # Move to the end of the word.
    while 0 <= i < n and g.isWordChar(s[i]):
        i += 1
    if select:
        w.setSelectionRange(i1,i)
    return i1,i
#@+node:ekr.20050920084036.140: *6* movePastClose & helper
def movePastClose (self,event):
    '''Move the cursor past the closing parenthesis.'''
    self.movePastCloseHelper(event,extend=False)

def movePastCloseExtendSelection (self,event):
    '''Extend the selection by moving the cursor past the closing parenthesis.'''
    self.movePastCloseHelper(event,extend=True)
#@+node:ekr.20051218171457: *7* movePastCloseHelper
def movePastCloseHelper (self,event,extend):

    c = self.c ; w = self.editWidget(event)
    if not w: return

    c.widgetWantsFocusNow(w)
    s = w.getAllText()
    ins = w.getInsertPoint()
    # Scan backwards for i,j.
    i = ins
    while i >= 0 and s[i] != '\n':
        if s[i] == '(': break
        i -= 1
    else: return
    j = ins
    while j >= 0 and s[j] != '\n':
        if s[j] == '(': break
        j -= 1
    if i < j: return
    # Scan forward for i2,j2.
    i2 = ins
    while i2 < len(s) and s[i2] != '\n':
        if s[i2] == ')': break
        i2 += 1
    else: return
    j2 = ins
    while j2 < len(s) and s[j2] != '\n':
        if s[j2] == ')': break
        j2 += 1
    if i2 > j2: return

    self.moveToHelper(event,i2+1,extend)
#@+node:ekr.20100109094541.6231: *6* moveWithinLineHelper
def moveWithinLineHelper (self,event,spot,extend):

    w = self.editWidget(event)
    if not w: return

    # Bug fix: 2012/02/28: don't use the Qt end-line logic:
    # it apparently does not work for wrapped lines.
    if hasattr(w,'leoMoveCursorHelper') and spot != 'end-line':
        extend = extend or self.extendMode
        w.leoMoveCursorHelper(kind=spot,extend=extend)
    else:
        s = w.getAllText()
        ins = w.getInsertPoint()
        i,j = g.getLine(s,ins)
        if spot == 'start-line':
            self.moveToHelper(event,i,extend=extend)
        elif spot == 'end-line':
            # Bug fix: 2011/11/13: Significant in external tests.
            if g.match(s,j-1,'\n'): j -= 1
            self.moveToHelper(event,j,extend=extend)
        else:
            g.trace('can not happen: bad spot: %s' % spot)
#@+node:ekr.20090530181848.6034: *6* pages & helper
def backPage (self,event):
    '''Move the cursor back one page,
    extending the selection if in extend mode.'''
    self.movePageHelper(event,kind='back',extend=False)

def backPageExtendSelection (self,event):
    '''Extend the selection by moving the cursor back one page.'''
    self.movePageHelper(event,kind='back',extend=True)

def forwardPage (self,event):
    '''Move the cursor forward one page,
    extending the selection if in extend mode.'''
    self.movePageHelper(event,kind='forward',extend=False)

def forwardPageExtendSelection (self,event):
    '''Extend the selection by moving the cursor forward one page.'''
    self.movePageHelper(event,kind='forward',extend=True)
#@+node:ekr.20090530181848.6035: *7* movePageHelper
def movePageHelper(self,event,kind,extend): # kind in back/forward.

    '''Move the cursor up/down one page, possibly extending the selection.'''

    trace = False and not g.unitTesting
    w = self.editWidget(event)
    if not w: return

    linesPerPage = 15 # To do.
    if hasattr(w,'leoMoveCursorHelper'):
        extend = extend or self.extendMode
        w.leoMoveCursorHelper(
            kind='page-down' if kind=='forward' else 'page-up',
            extend=extend,linesPerPage=linesPerPage)
        # w.seeInsertPoint()
        # c.frame.updateStatusLine()
        # w.rememberSelectionAndScroll()
    else:
        ins = w.getInsertPoint()
        s = w.getAllText()
        lines = g.splitLines(s)
        row,col = g.convertPythonIndexToRowCol(s,ins)
        row2 = max(0,row-linesPerPage) if kind=='back' else min(row+linesPerPage,len(lines)-1)
        if row == row2: return
        spot = g.convertRowColToPythonIndex(s,row2,col,lines=lines)
        if trace: g.trace('spot',spot,'row2',row2)
        self.extendHelper(w,extend,spot,upOrDown=True)
#@+node:ekr.20050920084036.102: *6* paragraphs & helpers
def backwardParagraph (self,event):
    '''Move the cursor to the previous paragraph.'''
    self.backwardParagraphHelper (event,extend=False)

def backwardParagraphExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the previous paragraph.'''
    self.backwardParagraphHelper (event,extend=True)

def forwardParagraph (self,event):
    '''Move the cursor to the next paragraph.'''
    self.forwardParagraphHelper(event,extend=False)

def forwardParagraphExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the next paragraph.'''
    self.forwardParagraphHelper(event,extend=True)
#@+node:ekr.20051218133207: *7* backwardParagraphHelper
def backwardParagraphHelper (self,event,extend):

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    i,j = w.getSelectionRange()
    # A hack for wx gui: set the insertion point to the end of the selection range.
    if g.app.unitTesting:
        w.setInsertPoint(j)
    i,j = g.getLine(s,j)
    line = s[i:j]

    if line.strip():
        # Find the start of the present paragraph.
        while i > 0:
            i,j = g.getLine(s,i-1)
            line = s[i:j]
            if not line.strip(): break

    # Find the end of the previous paragraph.
    while i > 0:
        i,j = g.getLine(s,i-1)
        line = s[i:j]
        if line.strip():
            i = j-1 ; break

    self.moveToHelper(event,i,extend)
#@+node:ekr.20051218133207.1: *7* forwardParagraphHelper
def forwardParagraphHelper (self,event,extend):

    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    line = s[i:j]

    if line.strip(): # Skip past the present paragraph.
        self.selectParagraphHelper(w,i)
        i,j = w.getSelectionRange()
        j += 1

    # Skip to the next non-blank line.
    i = j
    while j < len(s):
        i,j = g.getLine(s,j)
        line = s[i:j]
        if line.strip(): break

    w.setInsertPoint(ins) # Restore the original insert point.
    self.moveToHelper(event,i,extend)
#@+node:ekr.20061111223516: *6* selectAllText (leoEditCommands)
def selectAllText (self,event):

    '''Select all text.'''

    c,k = self.c,self.c.k
    w = self.editWidget(event)
    if not w: return
    # Bug fix 2013/12/13: Special case the minibuffer.
    if w == k.w:
        k.selectAll()
    else:
        isTextWrapper = g.isTextWrapper(w)
        if w and isTextWrapper:
            return w.selectAllText()
#@+node:ekr.20050920084036.131: *6* sentences & helpers
def backSentence (self,event):
    '''Move the cursor to the previous sentence.'''
    self.backSentenceHelper(event,extend=False)

def backSentenceExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the previous sentence.'''
    self.backSentenceHelper(event,extend=True)

def forwardSentence (self,event):
    '''Move the cursor to the next sentence.'''
    self.forwardSentenceHelper(event,extend=False)

def forwardSentenceExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the next sentence.'''
    self.forwardSentenceHelper(event,extend=True)
#@+node:ekr.20051213094517: *7* backSentenceHelper
def backSentenceHelper (self,event,extend):

    c = self.c
    w = self.editWidget(event)
    if not w: return

    c.widgetWantsFocusNow(w)
    s = w.getAllText()
    ins = w.getInsertPoint()

    # Find the starting point of the scan.
    i = ins
    i -= 1 # Ensure some progress.
    if i < 0:
        return

    # Tricky.
    if s[i] == '.':
        i -= 1
    while i >= 0 and s[i] in ' \n':
        i -= 1
    if i >= ins:
        i -= 1
    if i >= len(s):
        i -= 1
    if i <= 0:
        return
    if s[i] == '.':
        i -= 1

    # Scan backwards to the end of the paragraph.
    # Stop at empty lines.
    # Skip periods within words.
    # Stop at sentences ending in non-periods.
    end = False
    while not end and i >= 0:
        progress = i
        if s[i] == '.':
            # Skip periods surrounded by letters/numbers
            if i > 0 and s[i-1].isalnum() and s[i+1].isalnum():
                i -= 1
            else:
                i += 1
                while i < len(s) and s[i] in ' \n':
                    i += 1
                i -= 1
                break
        elif s[i] == '\n':
            j = i-1
            while j >= 0:
                if s[j] == '\n':
                    # Don't include first newline.
                    end = True ; break # found blank line.
                elif s[j] == ' ':
                    j -= 1
                else:
                    i -= 1 ; break # no blank line found.
            else:
                # No blank line found.
                i -= 1
        else:
            i -= 1
        assert end or progress > i
    i += 1

    if i < ins:
        self.moveToHelper(event,i,extend)
#@+node:ekr.20050920084036.137: *7* forwardSentenceHelper
def forwardSentenceHelper (self,event,extend):

    c = self.c
    w = self.editWidget(event)
    if not w: return

    c.widgetWantsFocusNow(w)

    s = w.getAllText()
    ins = w.getInsertPoint()
    if ins >= len(s): return

    # Find the starting point of the scan.
    i = ins
    if i+1 < len(s) and s[i+1] == '.':
        i += 1
    if s[i] == '.':
        i += 1
    else:
        while i < len(s) and s[i] in ' \n':
            i += 1
        i -= 1
    if i <= ins:
        i += 1
    if i >= len(s):
        return

    # Scan forward to the end of the paragraph.
    # Stop at empty lines.
    # Skip periods within words.
    # Stop at sentences ending in non-periods.
    end = False
    while not end and i < len(s):
        progress = i
        if s[i] == '.':
            # Skip periods surrounded by letters/numbers
            if 0 < i < len(s) and s[i-1].isalnum() and s[i+1].isalnum():
                i += 1
            else:
                i += 1 ; break # Include the paragraph.
        elif s[i] == '\n':
            j = i+1
            while j < len(s):
                if s[j] == '\n':
                    # Don't include first newline.
                    end = True ; break # found blank line.
                elif s[j] == ' ':
                    j += 1
                else:
                    i += 1 ; break # no blank line found.
            else:
                # No blank line found.
                i += 1
        else:
            i += 1
        assert end or progress < i

    i = min(i,len(s))
    if i > ins:
        self.moveToHelper(event,i,extend)
#@+node:ekr.20100109094541.6232: *6* within lines
def beginningOfLine (self,event):
    '''Move the cursor to the start of the line, extending the selection if in extend mode.'''
    self.moveWithinLineHelper(event,'start-line',extend=False)

def beginningOfLineExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the start of the line.'''
    self.moveWithinLineHelper(event,'start-line',extend=True)

def endOfLine (self,event):
    '''Move the cursor to the end of the line, extending the selection if in extend mode.'''
    self.moveWithinLineHelper(event,'end-line',extend=False)

def endOfLineExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the end of the line.'''
    self.moveWithinLineHelper(event,'end-line',extend=True)
#@+node:ekr.20050920084036.149: *6* words & helper
def backwardWord (self,event):
    '''Move the cursor to the previous word.'''
    self.moveWordHelper(event,extend=False,forward=False)

def backwardWordExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the previous word.'''
    self.moveWordHelper(event,extend=True,forward=False)

def forwardEndWord (self,event): # New in Leo 4.4.2
    '''Move the cursor to the next word.'''
    self.moveWordHelper(event,extend=False,forward=True,end=True)

def forwardEndWordExtendSelection (self,event): # New in Leo 4.4.2
    '''Extend the selection by moving the cursor to the next word.'''
    self.moveWordHelper(event,extend=True,forward=True,end=True)

def forwardWord (self,event):
    '''Move the cursor to the next word.'''
    self.moveWordHelper(event,extend=False,forward=True)

def forwardWordExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the end of the next word.'''
    self.moveWordHelper(event,extend=True,forward=True)

def backwardWordSmart (self,event):
    '''Move the cursor to the beginning of the current or the end of the previous word.'''
    self.moveWordHelper(event,extend=False,forward=False,smart=True)

def backwardWordSmartExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the beginning of the current
    or the end of the previous word.'''
    self.moveWordHelper(event,extend=True,forward=False,smart=True)

def forwardWordSmart (self,event):
    '''Move the cursor to the end of the current or the beginning of the next word.'''
    self.moveWordHelper(event,extend=False,forward=True,smart=True)

def forwardWordSmartExtendSelection (self,event):
    '''Extend the selection by moving the cursor to the end of the current
    or the beginning of the next word.'''
    self.moveWordHelper(event,extend=True,forward=True,smart=True)
#@+node:ekr.20051218121447: *7* moveWordHelper
def moveWordHelper (self,event,extend,forward,end=False,smart=False):

    '''Move the cursor to the next/previous word.
    The cursor is placed at the start of the word unless end=True'''

    c = self.c
    w = self.editWidget(event)
    if not w: return

    c.widgetWantsFocusNow(w)
    s = w.getAllText() ; n = len(s)
    i = w.getInsertPoint()

    # pylint: disable=anomalous-backslash-in-string
    alphanumeric_re = re.compile("\w")
    whitespace_re = re.compile("\s")
    simple_whitespace_re = re.compile("[ \t]")

    def is_alphanumeric(c):
        return alphanumeric_re.match(c) is not None
    def is_whitespace(c):
        return whitespace_re.match(c) is not None
    def is_simple_whitespace(c):
        return simple_whitespace_re.match(c) is not None
    def is_line_break(c):
        return is_whitespace(c) and not is_simple_whitespace(c)
    def is_special(c):
        return not is_alphanumeric(c) and not is_whitespace(c)

    def seek_until_changed(i, match_function, step):
        while 0 <= i < n and match_function(s[i]):
            i += step
        return i

    def seek_word_end(i): return seek_until_changed(i,is_alphanumeric,1)
    def seek_word_start(i): return seek_until_changed(i,is_alphanumeric,-1)

    def seek_simple_whitespace_end(i): return seek_until_changed(i,is_simple_whitespace,1)
    def seek_simple_whitespace_start(i): return seek_until_changed(i,is_simple_whitespace,-1)

    def seek_special_end(i): return seek_until_changed(i,is_special,1)
    def seek_special_start(i): return seek_until_changed(i,is_special,-1)

    if smart:
        if forward:
            if 0 <= i < n:
                if is_alphanumeric(s[i]):
                    i = seek_word_end(i)
                    i = seek_simple_whitespace_end(i)
                elif is_simple_whitespace(s[i]):
                    i = seek_simple_whitespace_end(i)
                elif is_special(s[i]):
                    i = seek_special_end(i)
                    i = seek_simple_whitespace_end(i)
                else:
                    i += 1  # e.g. for newlines
        else:
            i -= 1  # Shift cursor temporarily by -1 to get easy read access to the prev. char
            if 0 <= i < n:
                if is_alphanumeric(s[i]):
                    i = seek_word_start(i)
                    # Do not seek further whitespace here
                elif is_simple_whitespace(s[i]):
                    i = seek_simple_whitespace_start(i)
                elif is_special(s[i]):
                    i = seek_special_start(i)
                    # Do not seek further whitespace here
                else:
                    i -= 1  # e.g. for newlines
            i += 1
    else:
        if forward:
            # Unlike backward-word moves, there are two options...
            if end:
                while 0 <= i < n and not g.isWordChar(s[i]):
                    i += 1
                while 0 <= i < n and g.isWordChar(s[i]):
                    i += 1
            else:
                while 0 <= i < n and g.isWordChar(s[i]):
                    i += 1
                while 0 <= i < n and not g.isWordChar(s[i]):
                    i += 1
        else:
            i -= 1
            while 0 <= i < n and not g.isWordChar(s[i]):
                i -= 1
            while 0 <= i < n and g.isWordChar(s[i]):
                i -= 1

    self.moveToHelper(event,i,extend)
#@+node:ekr.20050920084036.95: *5* paragraph...
@others
#@+node:ekr.20050920084036.99: *6* backwardKillParagraph
def backwardKillParagraph (self,event):

    '''Kill the previous paragraph.'''

    k = self.k ; c = k.c ; w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType='backward-kill-paragraph')
    try:
        self.backwardParagraphHelper(event,extend=True)
        i,j = w.getSelectionRange()
        if i > 0: i = min(i+1,j)
        c.killBufferCommands.kill(event,i,j,undoType=None)
        w.setSelectionRange(i,i,insert=i)
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.100: *6* fillRegion
def fillRegion (self,event):
    '''Fill all paragraphs in the selected text.'''
    c = self.c ; p = c.p ; undoType = 'fill-region'
    w = self.editWidget(event)
    i,j = w.getSelectionRange()
    c.undoer.beforeChangeGroup(p,undoType)
    while 1:
        progress = w.getInsertPoint()
        c.reformatParagraph(event,undoType='reformat-paragraph')
        ins = w.getInsertPoint()
        s = w.getAllText()
        w.setInsertPoint(ins)
        if progress >= ins or ins >= j or ins >= len(s):
            break
    c.undoer.afterChangeGroup(p,undoType)
#@+node:ekr.20050920084036.104: *6* fillRegionAsParagraph
def fillRegionAsParagraph (self,event):

    '''Fill the selected text.'''

    w = self.editWidget(event)
    if not w or not self._chckSel(event): return

    self.beginCommand(undoType='fill-region-as-paragraph')

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.103: *6* fillParagraph
def fillParagraph( self, event ):

    '''Fill the selected paragraph'''

    w = self.editWidget(event)
    if not w: return

    # Clear the selection range.
    i,j = w.getSelectionRange()
    w.setSelectionRange(i,i,insert=i)

    self.c.reformatParagraph(event)
#@+node:ekr.20050920084036.98: *6* killParagraph
def killParagraph (self,event):

    '''Kill the present paragraph.'''

    k = self.k ; c = k.c ; w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType='kill-paragraph')
    try:
        self.extendToParagraph(event)
        i,j = w.getSelectionRange()
        c.killBufferCommands.kill(event,i,j,undoType=None)
        w.setSelectionRange(i,i,insert=i)
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.96: *6* extend-to-paragraph & helper
def extendToParagraph (self,event):

    '''Select the paragraph surrounding the cursor.'''

    w = self.editWidget(event)
    if not w: return
    s = w.getAllText() ; ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    line = s[i:j]

    # Find the start of the paragraph.
    if line.strip(): # Search backward.
        while i > 0:
            i2,j2 = g.getLine(s,i-1)
            line = s[i2:j2]
            if line.strip(): i = i2
            else: break # Use the previous line.
    else: # Search forward.
        while j < len(s):
            i,j = g.getLine(s,j)
            line = s[i:j]
            if line.strip(): break
        else: return

    # Select from i to the end of the paragraph.
    self.selectParagraphHelper(w,i)
#@+node:ekr.20050920084036.97: *7* selectParagraphHelper
def selectParagraphHelper (self,w,start):

    '''Select from start to the end of the paragraph.'''

    s = w.getAllText()
    i1,j = g.getLine(s,start)
    while j < len(s):
        i,j2 = g.getLine(s,j)
        line = s[i:j2]
        if line.strip(): j = j2
        else: break

    j = max(start,j-1)
    w.setSelectionRange(i1,j,insert=j)
#@+node:ekr.20050920084036.105: *5* region...
@others
#@+node:ekr.20050920084036.108: *6* tabIndentRegion (indent-rigidly)
def tabIndentRegion (self,event):

    '''Insert a hard tab at the start of each line of the selected text.'''

    w = self.editWidget(event)
    if not w or not self._chckSel(event): return

    self.beginCommand(undoType='indent-rigidly')

    s = w.getAllText()
    i1,j1 = w.getSelectionRange()
    i,junk = g.getLine(s,i1)
    junk,j = g.getLine(s,j1)

    lines = g.splitlines(s[i:j])
    n = len(lines)
    lines = g.joinLines(['\t' + line for line in lines])
    s = s[:i] + lines + s[j:]
    w.setAllText(s)

    # Retain original row/col selection.
    w.setSelectionRange(i1,j1+n,insert=j1+n)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.109: *6* countRegion
def countRegion (self,event):

    '''Print the number of lines and characters in the selected text.'''

    k = self.k
    w = self.editWidget(event)
    if not w: return

    txt = w.getSelectedText()
    lines = 1 ; chars = 0
    for z in txt:
        if z == '\n': lines += 1
        else:         chars += 1

    k.setLabelGrey('Region has %s lines, %s character%s' % (
        lines,chars,'' if chars==1 else 's'))
#@+node:ekr.20060417183606: *6* moveLinesDown
def moveLinesDown (self,event):

    '''Move all lines containing any selected text down one line,
    moving to the next node if the lines are the last lines of the body.'''

    c = self.c ; w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    sel_1,sel_2 = w.getSelectionRange()
    insert_pt = w.getInsertPoint() # 2011/04/01
    i,junk = g.getLine(s,sel_1)
    i2,j   = g.getLine(s,sel_2)
    lines  = s[i:j]

    # Select from start of the first line to the *start* of the last line.
    # This prevents selection creep.
    # n = i2-i
    # g.trace('moveLinesDown:',repr('%s[[%s|%s|%s]]%s' % (
    #    s[i-20:i], s[i:sel_1], s[sel_1:sel_2], s[sel_2:j], s[j:j+20])))
    self.beginCommand(undoType='move-lines-down')
    changed = False
    try:
        if j < len(s):
            next_i,next_j = g.getLine(s,j) # 2011/04/01: was j+1
            next_line = s[next_i:next_j]
            n2 = next_j-next_i
            w.delete(i,next_j)
            if next_line.endswith('\n'):
                # Simply swap positions with next line
                new_lines = next_line+lines
            else:
                # Last line of the body to be moved up doesn't end in a newline
                # while we have to remove the newline from the line above moving down.
                new_lines = next_line+'\n'+lines[:-1]
                n2 += 1
            w.insert(i,new_lines)
            w.setSelectionRange(sel_1+n2,sel_2+n2,insert=insert_pt+n2)
            changed = True
            # Fix bug 799695: colorizer bug after move-lines-up into a docstring
            c.recolor_now(incremental=False)
    finally:
        self.endCommand(changed=changed,setLabel=True)
#@+node:ekr.20060417183606.1: *6* moveLinesUp
def moveLinesUp (self,event):

    '''Move all lines containing any selected text up one line,
    moving to the previous node as needed.'''

    c = self.c ; w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    sel_1,sel_2 = w.getSelectionRange()
    insert_pt = w.getInsertPoint() # 2011/04/01     
    i,junk = g.getLine(s,sel_1)
    i2,j   = g.getLine(s,sel_2)
    lines  = s[i:j]
    # g.trace('moveLinesUp:',repr('%s[[%s|%s|%s]]%s' % (
    #    s[max(0,i-20):i], s[i:sel_1], s[sel_1:sel_2], s[sel_2:j], s[j:j+20])))
    self.beginCommand(undoType='move-lines-up')
    changed = False
    try:
        if i>0:
            prev_i,prev_j = g.getLine(s,i-1)
            prev_line = s[prev_i:prev_j]
            n2 = prev_j - prev_i
            w.delete(prev_i,j)
            if lines.endswith('\n'):
                # Simply swap positions with next line
                new_lines = lines+prev_line
            else:
                # Lines to be moved up don't end in a newline while the
                # previous line going down needs its newline taken off.
                new_lines = lines+'\n'+prev_line[:-1]
            w.insert(prev_i,new_lines)
            w.setSelectionRange(sel_1-n2,sel_2-n2,insert=insert_pt-n2)
            changed = True
            # Fix bug 799695: colorizer bug after move-lines-up into a docstring
            c.recolor_now(incremental=False)
    finally:
        self.endCommand(changed=changed,setLabel=True)
#@+node:ekr.20050920084036.110: *6* reverseRegion
def reverseRegion (self,event):

    '''Reverse the order of lines in the selected text.'''

    w = self.editWidget(event)
    if not w or not self._chckSel(event): return

    self.beginCommand(undoType='reverse-region')

    s = w.getAllText()
    i1,j1 = w.getSelectionRange()
    i,junk = g.getLine(s,i1)
    junk,j = g.getLine(s,j1)

    txt = s[i:j]
    aList = txt.split('\n')
    aList.reverse()
    txt = '\n'.join(aList) + '\n'

    w.setAllText(s[:i1] + txt + s[j1:])
    ins = i1 + len(txt) - 1
    w.setSelectionRange(ins,ins,insert=ins)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.111: *6* up/downCaseRegion & helper
def downCaseRegion (self,event):
    '''Convert all characters in the selected text to lower case.'''
    self.caseHelper(event,'low','downcase-region')

def toggleCaseRegion (self,event):
    '''Toggle the case of all characters in the selected text.'''
    self.caseHelper(event,'toggle','toggle-case-region')

def upCaseRegion (self,event):
    '''Convert all characters in the selected text to UPPER CASE.'''
    self.caseHelper(event,'up','upcase-region')

def caseHelper (self,event,way,undoType):

    w = self.editWidget(event)
    if not w or not w.hasSelection():
        return
    self.beginCommand(undoType=undoType)
    s = w.getAllText()
    i,j = w.getSelectionRange()
    ins = w.getInsertPoint()
    s2 = s[i:j]
    if way == 'low':
        sel = s2.lower()
    elif way == 'up':
        sel = s2.upper()
    else:
        assert way == 'toggle'
        sel = s2.swapcase()
    s2 = s[:i] + sel + s[j:]
    # g.trace('sel',repr(sel),'s2',repr(s2))
    changed = s2 != s
    if changed:
        w.setAllText(s2)
        w.setSelectionRange(i,j,insert=ins)
    self.endCommand(changed=changed,setLabel=True)
#@+node:ekr.20060309060654: *5* scrolling...
#@+node:ekr.20050920084036.116: *6* scrollUp/Down & helper
def scrollDownHalfPage (self,event):
    '''Scroll the presently selected pane down one lline.'''
    self.scrollHelper(event,'down','half-page')

def scrollDownLine (self,event):
    '''Scroll the presently selected pane down one lline.'''
    self.scrollHelper(event,'down','line')

def scrollDownPage (self,event):
    '''Scroll the presently selected pane down one page.'''
    self.scrollHelper(event,'down','page')

def scrollUpHalfPage (self,event):
    '''Scroll the presently selected pane down one lline.'''
    self.scrollHelper(event,'up','half-page')

def scrollUpLine (self,event):
    '''Scroll the presently selected pane up one page.'''
    self.scrollHelper(event,'up','line')

def scrollUpPage (self,event):
    '''Scroll the presently selected pane up one page.'''
    self.scrollHelper(event,'up','page')
#@+node:ekr.20060113082917: *7* scrollHelper (leoEditCommands)
def scrollHelper (self,event,direction,distance):

    '''Scroll the present pane up or down one page
    kind is in ('up/down-half-page/line/page)'''

    w = event and event.w
    if w and hasattr(w,'scrollDelegate'):
        kind = direction + '-' + distance
        w.scrollDelegate(kind)
#@+node:ekr.20060309060654.1: *6* scrollOutlineUp/Down/Line/Page
def scrollOutlineDownLine (self,event=None):
    '''Scroll the outline pane down one line.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('down-line')
    elif hasattr(tree.canvas,'leo_treeBar'):
        a,b = tree.canvas.leo_treeBar.get()
        if b < 1.0: tree.canvas.yview_scroll(1,"unit")

def scrollOutlineDownPage (self,event=None):
    '''Scroll the outline pane down one page.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('down-page')
    elif hasattr(tree.canvas,'leo_treeBar'):
        a,b = tree.canvas.leo_treeBar.get()
        if b < 1.0: tree.canvas.yview_scroll(1,"page")

def scrollOutlineUpLine (self,event=None):
    '''Scroll the outline pane up one line.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('up-line')
    elif hasattr(tree.canvas,'leo_treeBar'):
        a,b = tree.canvas.leo_treeBar.get()
        if a > 0.0: tree.canvas.yview_scroll(-1,"unit")

def scrollOutlineUpPage (self,event=None):
    '''Scroll the outline pane up one page.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('up-page')
    elif hasattr(tree.canvas,'leo_treeBar'):
        a,b = tree.canvas.leo_treeBar.get()
        if a > 0.0: tree.canvas.yview_scroll(-1,"page")
#@+node:ekr.20060726154531: *6* scrollOutlineLeftRight
def scrollOutlineLeft (self,event=None):
    '''Scroll the outline left.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('left')
    elif hasattr(tree.canvas,'xview_scroll'):
        tree.canvas.xview_scroll(1,"unit")

def scrollOutlineRight (self,event=None):
    '''Scroll the outline left.'''
    c = self.c ; tree = c.frame.tree
    if hasattr(tree,'scrollDelegate'):
        tree.scrollDelegate('right')
    elif hasattr(tree.canvas,'xview_scroll'):
        tree.canvas.xview_scroll(-1,"unit")
#@+node:ekr.20050920084036.117: *5* sort...
@nocolor
@color
@
XEmacs provides several commands for sorting text in a buffer.  All
operate on the contents of the region (the text between point and the
mark).  They divide the text of the region into many "sort records",
identify a "sort key" for each record, and then reorder the records
using the order determined by the sort keys.  The records are ordered so
that their keys are in alphabetical order, or, for numerical sorting, in
numerical order.  In alphabetical sorting, all upper-case letters `A'
through `Z' come before lower-case `a', in accordance with the ASCII
character sequence.

   The sort commands differ in how they divide the text into sort
records and in which part of each record they use as the sort key.
Most of the commands make each line a separate sort record, but some
commands use paragraphs or pages as sort records.  Most of the sort
commands use each entire sort record as its own sort key, but some use
only a portion of the record as the sort key.

`M-x sort-lines'
     Divide the region into lines and sort by comparing the entire text
     of a line.  A prefix argument means sort in descending order.

`M-x sort-paragraphs'
     Divide the region into paragraphs and sort by comparing the entire
     text of a paragraph (except for leading blank lines).  A prefix
     argument means sort in descending order.

`M-x sort-pages'
     Divide the region into pages and sort by comparing the entire text
     of a page (except for leading blank lines).  A prefix argument
     means sort in descending order.

`M-x sort-fields'
     Divide the region into lines and sort by comparing the contents of
     one field in each line.  Fields are defined as separated by
     whitespace, so the first run of consecutive non-whitespace
     characters in a line constitutes field 1, the second such run
     constitutes field 2, etc.

     You specify which field to sort by with a numeric argument: 1 to
     sort by field 1, etc.  A negative argument means sort in descending
     order.  Thus, minus 2 means sort by field 2 in reverse-alphabetical
     order.

`M-x sort-numeric-fields'
     Like `M-x sort-fields', except the specified field is converted to
     a number for each line and the numbers are compared.  `10' comes
     before `2' when considered as text, but after it when considered
     as a number.

`M-x sort-columns'
     Like `M-x sort-fields', except that the text within each line used
     for comparison comes from a fixed range of columns.  An explanation
     is given below.

   For example, if the buffer contains:

     On systems where clash detection (locking of files being edited) is
     implemented, XEmacs also checks the first time you modify a buffer
     whether the file has changed on disk since it was last visited or
     saved.  If it has, you are asked to confirm that you want to change
     the buffer.

then if you apply `M-x sort-lines' to the entire buffer you get:

     On systems where clash detection (locking of files being edited) is
     implemented, XEmacs also checks the first time you modify a buffer
     saved.  If it has, you are asked to confirm that you want to change
     the buffer.
     whether the file has changed on disk since it was last visited or

where the upper case `O' comes before all lower case letters.  If you
apply instead `C-u 2 M-x sort-fields' you get:

     saved.  If it has, you are asked to confirm that you want to change
     implemented, XEmacs also checks the first time you modify a buffer
     the buffer.
     On systems where clash detection (locking of files being edited) is
     whether the file has changed on disk since it was last visited or

where the sort keys were `If', `XEmacs', `buffer', `systems', and `the'.

   `M-x sort-columns' requires more explanation.  You specify the
columns by putting point at one of the columns and the mark at the other
column.  Because this means you cannot put point or the mark at the
beginning of the first line to sort, this command uses an unusual
definition of `region': all of the line point is in is considered part
of the region, and so is all of the line the mark is in.

   For example, to sort a table by information found in columns 10 to
15, you could put the mark on column 10 in the first line of the table,
and point on column 15 in the last line of the table, and then use this
command.  Or you could put the mark on column 15 in the first line and
point on column 10 in the last line.

   This can be thought of as sorting the rectangle specified by point
and the mark, except that the text on each line to the left or right of
the rectangle moves along with the text inside the rectangle.  *Note
Rectangles::.

#@+node:ekr.20050920084036.118: *6* sortLines commands
def reverseSortLinesIgnoringCase(self,event):
    '''Sort the selected lines in reverse order, ignoring case.'''
    return self.sortLines(event,ignoreCase=True,reverse=True)

def reverseSortLines(self,event):
    '''Sort the selected lines in reverse order.'''
    return self.sortLines(event,reverse=True)

def sortLinesIgnoringCase(self,event):
    '''Sort the selected lines, ignoring case.'''
    return self.sortLines(event,ignoreCase=True)

def sortLines (self,event,ignoreCase=False,reverse=False):
    '''Sort the selected lines.'''
    w = self.editWidget(event)
    if not self._chckSel(event): return

    undoType = 'reverse-sort-lines' if reverse else 'sort-lines'
    self.beginCommand(undoType=undoType)
    try:
        s = w.getAllText()
        sel_1,sel_2 = w.getSelectionRange()
        ins = w.getInsertPoint()
        i,junk = g.getLine(s,sel_1)
        junk,j = g.getLine(s,sel_2)
        s2 = s[i:j]
        if not s2.endswith('\n'): s2 = s2+'\n'
        aList = g.splitLines(s2)
        def lower(s):
            if ignoreCase: return s.lower()
            else: return s
        aList.sort(key=lower)
            # key is a function that extracts args.
        if reverse:
            aList.reverse()
        s = g.joinLines(aList)
        w.delete(i,j)
        w.insert(i,s)
        w.setSelectionRange(sel_1,sel_2,insert=ins)
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.119: *6* sortColumns
def sortColumns (self,event):

    '''Sort lines of selected text using only lines in the given columns to do the comparison.'''

    w = self.editWidget(event)
    if not self._chckSel(event): return
    self.beginCommand(undoType='sort-columns')
    try:
        s = w.getAllText()
        sel_1,sel_2 = w.getSelectionRange()
        sint1,sint2 = g.convertPythonIndexToRowCol(s,sel_1)
        sint3,sint4 = g.convertPythonIndexToRowCol(s,sel_2)
        sint1 += 1 ; sint3 += 1
        i,junk = g.getLine(s,sel_1)
        junk,j = g.getLine(s,sel_2)
        txt = s[i:j]
        columns = [w.get('%s.%s' % (z,sint2),'%s.%s' % (z,sint4))
            for z in range(sint1,sint3+1)]
        aList = g.splitLines(txt)
        zlist = list(zip(columns,aList))
        zlist.sort()
        s = g.joinLines([z[1] for z in zlist])
        w.delete(i,j)
        w.insert(i,s)
        w.setSelectionRange(sel_1,sel_1+len(s),insert=sel_1+len(s))
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.120: *6* sortFields
def sortFields (self,event,which=None):

    '''Divide the selected text into lines and sort by comparing the contents of
     one field in each line. Fields are defined as separated by whitespace, so
     the first run of consecutive non-whitespace characters in a line
     constitutes field 1, the second such run constitutes field 2, etc.

     You specify which field to sort by with a numeric argument: 1 to sort by
     field 1, etc. A negative argument means sort in descending order. Thus,
     minus 2 means sort by field 2 in reverse-alphabetical order.'''

    w = self.editWidget(event)
    if not w or not self._chckSel(event): return

    self.beginCommand(undoType='sort-fields')

    s = w.getAllText()
    ins = w.getInsertPoint()
    r1,r2,r3,r4 = self.getRectanglePoints(w)
    i,junk = g.getLine(s,r1)
    junk,j = g.getLine(s,r4)
    txt = s[i:j] # bug reported by pychecker.
    txt = txt.split('\n')
    fields = []
    fn = r'\w+'
    frx = re.compile(fn)
    for line in txt:
        f = frx.findall(line)
        if not which:
            fields.append(f[0])
        else:
            i = int(which)
            if len(f) < i: return
            i = i-1
            fields.append(f[i])
    nz = zip(fields,txt)
    nz.sort()
    w.delete(i,j)
    int1 = i
    for z in nz:
        w.insert('%s.0' % int1,'%s\n' % z[1])
        int1 = int1 + 1
    w.setInsertPoint(ins)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.121: *5* swap/transpose...
#@+node:ekr.20050920084036.122: *6* transposeLines
def transposeLines (self,event):

    '''Transpose the line containing the cursor with the preceding line.'''

    w = self.editWidget(event)
    if not w: return

    ins = w.getInsertPoint()
    s = w.getAllText()
    if not s.strip(): return

    i,j = g.getLine(s,ins)
    line1 = s[i:j]

    self.beginCommand(undoType='transpose-lines')

    if i == 0: # Transpose the next line.
        i2,j2 = g.getLine(s,j+1)
        line2 = s[i2:j2]
        w.delete(0,j2)
        w.insert(0,line2+line1)
        w.setInsertPoint(j2-1)
    else: # Transpose the previous line.
        i2,j2 = g.getLine(s,i-1)
        line2 = s[i2:j2]
        w.delete(i2,j)
        w.insert(i2,line1+line2)
        w.setInsertPoint(j-1)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20060529184652.1: *6* transposeWords
def transposeWords (self,event):

    '''Transpose the word at the cursor with the preceding or following word.'''

    trace = False and not g.unitTesting
    w = self.editWidget(event)
    if not w: return
    self.beginCommand(undoType='transpose-words')
    s = w.getAllText()
    i1,j1 = self.extendToWord(event,direction='back',select=False)
    s1 = s[i1:j1]
    if trace: g.trace(i1,j1,s1)
    if i1 > j1: i1,j1 = j1,i1

    # First, search backward.
    k = i1-1
    while k >= 0 and s[k].isspace():
        k -= 1
    changed = k > 0
    if changed:
        ws = s[k+1:i1]
        if trace: g.trace(repr(ws))
        w.setInsertPoint(k+1)
        i2,j2 = self.extendToWord(event,direction='back',select=False)
        s2 = s[i2:j2]
        if trace: g.trace(i2,j2,repr(s2))
        s3 = s[:i2] + s1 + ws + s2 + s[j1:]
        w.setAllText(s3)
        if trace: g.trace(s3)
        w.setSelectionRange(j1,j1,insert=j1)
    else:
        # Next search forward.
        k = j1+1
        while k < len(s) and s[k].isspace():
            k += 1
        changed = k < len(s)
        if changed:
            ws = s[j1:k]
            if trace: g.trace(repr(ws))
            w.setInsertPoint(k+1)
            i2,j2 = self.extendToWord(event,direction='forward',select=False)
            s2 = s[i2:j2]
            if trace: g.trace(i2,j2,repr(s2))
            s3 = s[:i1] + s2 + ws + s1 + s[j2:]
            w.setAllText(s3)
            if trace: g.trace(s3)
            w.setSelectionRange(j1,j1,insert=j1)

    self.endCommand(changed=changed,setLabel=True)
#@+node:ekr.20050920084036.124: *6* swapCharacters & transeposeCharacters
def swapCharacters (self,event):

    '''Swap the characters at the cursor.'''

    w = self.editWidget(event)
    if not w: return

    self.beginCommand(undoType='swap-characters')
    s = w.getAllText()
    i = w.getInsertPoint()
    if 0 < i < len(s):
        w.setAllText(s[:i-1] + s[i] + s[i-1] + s[i+1:])
        w.setSelectionRange(i,i,insert=i)
    self.endCommand(changed=True,setLabel=True)

transposeCharacters = swapCharacters
#@+node:ekr.20050920084036.126: *5* tabify & untabify
def tabify (self,event):
    '''Convert 4 spaces to tabs in the selected text.'''
    self.tabifyHelper (event,which='tabify')

def untabify (self,event):
    '''Convert tabs to 4 spaces in the selected text.'''
    self.tabifyHelper (event,which='untabify')

def tabifyHelper (self,event,which):

    w = self.editWidget(event)
    if not w or not w.hasSelection(): return

    self.beginCommand(undoType=which)
    i,end = w.getSelectionRange()
    txt = w.getSelectedText()
    if which == 'tabify':
        pattern = re.compile(' {4,4}') # Huh?
        ntxt = pattern.sub('\t',txt)
    else:
        pattern = re.compile('\t')
        ntxt = pattern.sub('    ',txt)
    w.delete(i,end)
    w.insert(i,ntxt)
    n = i + len(ntxt)
    w.setSelectionRange(n,n,insert=n)
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20110527105255.18384: *5* uA's (leoEditCommands)
#@+node:ekr.20110527105255.18387: *6* clearNodeUas & clearAllUas
def clearNodeUas (self,event=None):

    '''Clear the uA's in the selected VNode.'''

    if self.c.p:
        self.c.p.v.u = {}

def clearAllUas (self,event=None):

    '''Clear all uAs in the entire outline.'''

    for v in self.c.all_unique_nodes():
        v.u = {}
#@+node:ekr.20110527105255.18385: *6* printUas & printAllUas
def printAllUas (self,event=None):

    '''Print all uA's in the outline.'''

    g.es_print('Dump of uAs...')
    for v in self.c.all_unique_nodes():
        if v.u:
            self.printUas(v=v)

def printUas (self,event=None,v=None):

    '''Print the uA's in the selected node.'''

    c = self.c
    if v: d,h = v.u,v.h
    else: d,h = c.p.v.u,c.p.h
    g.es_print(h)
    keys = list(d.keys())
    keys.sort()
    n = 4
    for key in keys:
        n = max(len(key),n)
    for key in keys:
        pad = ' '*(len(key)-n)
        g.es_print('    %s%s: %s' % (pad,key,d.get(key)))
#@+node:ekr.20110527105255.18386: *6* setUa
def setUa (self,event):

    '''Prompt for the name and value of a uA, then set the uA in the present node.'''

    c,k = self.c,self.k
    tag = 'set-ua' ; state = k.getState(tag)
    if state == 0:
        w = self.editWidget(event) # sets self.w
        if w:
            k.setLabelBlue('Set uA: ')
            k.getArg(event,tag,1,self.setUa)
    elif state == 1:
        self.uaName = k.arg
        s = 'Set uA: %s To: ' % (self.uaName)
        k.setLabelBlue(s)
        k.getArg(event,tag,2,self.setUa,completion=False)
    else:
        assert state == 2,state
        val = k.arg
        d = c.p.v.u
        d[self.uaName] = val
        self.printUas()
        k.clearState()
        k.resetLabel()
        k.showStateAndMode()
#@+node:ekr.20050920084036.161: *4* EditFileCommandsClass
class EditFileCommandsClass (BaseEditCommandsClass):

    '''A class to load files into buffers and save buffers to files.'''

    @others
#@+node:ekr.20050920084036.162: *5*  ctor (EditFileCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.
#@+node:ekr.20050920084036.163: *5*  getPublicCommands (EditFileCommandsClass)
def getPublicCommands (self):

    return {
        'directory-make':           self.makeDirectory,
        'directory-remove':         self.removeDirectory,
        'file-compare-leo-files':   self.compareLeoFiles,
        'file-delete':              self.deleteFile,
        'file-diff-files':          self.diff, 
        'file-insert':              self.insertFile,
        'file-open-by-name':        self.openOutlineByName,
        'file-save':                self.saveFile
    }
#@+node:ekr.20070920104110: *5* compareLeoFiles
def compareLeoFiles (self,event):
    '''Compare two .leo files.'''
    trace = False and not g.unitTesting
    c = c1 = self.c
    w = c.frame.body.wrapper
    # Prompt for the file to be compared with the present outline.
    filetypes = [("Leo files", "*.leo"),("All files", "*"),]
    fileName = g.app.gui.runOpenFileDialog(
        title="Compare .leo Files",filetypes=filetypes,defaultextension='.leo')
    if not fileName: return
    # Read the file into the hidden commander.
    c2 = self.createHiddenCommander(fileName)
    if not c2: return
    # Compute the inserted, deleted and changed dicts.
    d1 = self.createFileDict(c1)
    d2 = self.createFileDict(c2)  
    inserted, deleted, changed = self.computeChangeDicts(d1,d2)
    if trace: self.dumpCompareNodes(fileName,c1.mFileName,inserted,deleted,changed)
    # Create clones of all inserted, deleted and changed dicts.
    self.createAllCompareClones(c1,c2,inserted,deleted,changed)
    # Fix bug 1231656: File-Compare-Leo-Files leaves other file open-count incremented.
    g.app.forgetOpenFile(fn=c2.fileName(),force=True)
    c2.frame.destroySelf()
    g.app.gui.set_focus(c,w)
#@+node:ekr.20070921072608: *6* computeChangeDicts
def computeChangeDicts (self,d1,d2):

    '''Compute inserted, deleted, changed dictionaries.
    
    New in Leo 4.11: show the nodes in the *invisible* file, d2, if possible.'''

    inserted = {}
    for key in d2:
        if not d1.get(key):
            inserted[key] = d2.get(key)
    deleted = {}
    for key in d1:
        if not d2.get(key):
            deleted[key] = d1.get(key)
    changed = {}
    for key in d1:
        if d2.get(key):
            p1 = d1.get(key)
            p2 = d2.get(key)
            if p1.h != p2.h or p1.b != p2.b:
                changed[key] = p2 # Show the node in the *other* file.
    return inserted, deleted, changed
#@+node:ekr.20070921072910: *6* createAllCompareClones & helper
def createAllCompareClones(self,c1,c2,inserted,deleted,changed):

    c = self.c # Always use the visible commander
    assert c == c1
    # Create parent node at the start of the outline.
    u = c.undoer ; undoType = 'Compare .leo Files'
    u.beforeChangeGroup(c.p,undoType)
    undoData = u.beforeInsertNode(c.p)
    parent = c.p.insertAfter()
    parent.setHeadString(undoType)
    u.afterInsertNode(parent,undoType,undoData,dirtyVnodeList=[])
    for d,kind in (
        (deleted,'not in %s' % c2.shortFileName()),
        (inserted,'not in %s' % c1.shortFileName()),
        (changed,'changed: as in %s' % c2.shortFileName()),
    ):
        self.createCompareClones(d,kind,parent)
    c.selectPosition(parent)
    u.afterChangeGroup(parent,undoType,reportFlag=True) 
    c.redraw()
#@+node:ekr.20070921074410: *7* createCompareClones
def createCompareClones (self,d,kind,parent):

    if d:
        c = self.c # Use the visible commander.
        parent = parent.insertAsLastChild()
        parent.setHeadString(kind)
        for key in d:
            p = d.get(key)
            if p.v.context == c:
                clone = p.clone()
                clone.moveToLastChildOf(parent)
            else:
                # Fix bug 1160660: File-Compare-Leo-Files creates "other file" clones.
                copy = p.copyTreeAfter()
                copy.moveToLastChildOf(parent)
                for p2 in copy.self_and_subtree():
                    p2.v.context = c
#@+node:ekr.20070921070101: *6* createHiddenCommander (EditFileCommandsClass)
def createHiddenCommander(self,fn):
    '''Read the file into a hidden commander (Similar to g.openWithFileName).'''
    import leo.core.leoCommands as leoCommands
    lm = g.app.loadManager
    c2 = leoCommands.Commands(fn,gui=g.app.nullGui)
    theFile = lm.openLeoOrZipFile(fn)
    if theFile:
        c2.fileCommands.openLeoFile(theFile,fn,
            readAtFileNodesFlag=True,silent=True)
        return c2
    else:
        return None
#@+node:ekr.20070921070101.1: *6* createFileDict
def createFileDict (self,c):
    '''Create a dictionary of all relevant positions in commander c.'''
    d = {}
    for p in c.all_positions():
        d[p.v.fileIndex] = p.copy()
    return d
#@+node:ekr.20070921072608.1: *6* dumpCompareNodes
def dumpCompareNodes (self,fileName1,fileName2,inserted,deleted,changed):

    for d,kind in (
        (inserted,'inserted (only in %s)' % (fileName1)),
        (deleted, 'deleted  (only in %s)' % (fileName2)),
        (changed, 'changed'),
    ):
        g.pr('\n',kind)
        for key in d:
            p = d.get(key)
            if g.isPython3:
                g.pr('%-32s %s' % (key,p.h))
            else:
                g.pr('%-32s %s' % (key,g.toEncodedString(p.h,'ascii')))
#@+node:ekr.20050920084036.164: *5* deleteFile (changed)
def deleteFile (self,event):

    '''Prompt for the name of a file and delete it.'''

    k = self.k ; state = k.getState('delete_file')

    if state == 0:
        k.setLabelBlue('Delete File: ')
        k.extendLabel(os.getcwd() + os.sep)
        k.getArg(event,'delete_file',1,self.deleteFile)
    else:
        k.keyboardQuit()
        k.clearState()
        try:
            os.remove(k.arg)
            k.setStatusLabel('Deleted: %s' % k.arg)
        except Exception:
            k.setStatusLabel('Not Deleted: %s' % k.arg)
#@+node:ekr.20050920084036.165: *5* diff (revise)
def diff (self,event):

    '''Creates a node and puts the diff between 2 files into it.'''

    w = self.editWidget(event)
    if not w: return
    fn = self.getReadableTextFile()
    if not fn: return
    fn2 = self.getReadableTextFile()
    if not fn2: return
    s1,e = g.readFileIntoString(fn)
    if s1 is None: return
    s2,e = g.readFileIntoString(fn2)
    if s2 is None: return

    # self.switchToBuffer(event,"*diff* of ( %s , %s )" % (name,name2))
    data = difflib.ndiff(s1,s2)
    idata = []
    for z in data:
        idata.append(z)
    w.delete(0,'end')
    w.insert(0,''.join(idata))
#@+node:ekr.20050920084036.166: *5* getReadableTextFile
def getReadableTextFile (self):

    fn = g.app.gui.runOpenFileDialog(
        title = 'Open Text File',
        filetypes = [("Text","*.txt"), ("All files","*")],
        defaultextension = ".txt")

    return fn
#@+node:ekr.20050920084036.167: *5* insertFile
def insertFile (self,event):

    '''Prompt for the name of a file and put the selected text into it.'''

    w = self.editWidget(event)
    if not w: return

    fn = self.getReadableTextFile()
    if not fn: return

    s,e = g.readFileIntoString(fn)
    if s is None: return

    self.beginCommand(undoType='insert-file')
    i = w.getInsertPoint()
    w.insert(i,s)
    w.seeInsertPoint()
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.168: *5* makeDirectory
def makeDirectory (self,event):
    '''Prompt for the name of a directory and create it.'''
    k = self.k
    state = k.getState('make_directory')
    if state == 0:
        k.setLabelBlue('Make Directory: ')
        k.extendLabel(os.getcwd() + os.sep)
        k.getArg(event,'make_directory',1,self.makeDirectory)
    else:
        k.keyboardQuit()
        k.clearState()
        try:
            os.mkdir(k.arg)
            k.setStatusLabel("Created: %s" % k.arg)
        except Exception:
            k.setStatusLabel("Not Create: %s" % k.arg)
#@+node:ekr.20060419123128: *5* openOutlineByName (EditFileCommandsClass)
def openOutlineByName (self,event):
    '''file-open-by-name: Prompt for the name of a Leo outline and open it.'''
    c,k = self.c,self.k
    fileName = ''.join(k.givenArgs)
    # Bug fix: 2012/04/09: only call g.openWithFileName if the file exists.
    if fileName and g.os_path_exists(fileName):
        g.openWithFileName(fileName,old_c=c)
    else:
        k.setLabelBlue('Open Leo Outline: ')
        k.getFileName(event,callback=self.openOutlineByNameFinisher)

def openOutlineByNameFinisher (self,fn):
    c = self.c
    if fn and g.os_path_exists(fn) and not g.os_path_isdir(fn):
        c2 = g.openWithFileName(fn,old_c=c)
        try:
            g.app.gui.runAtIdle(c2.treeWantsFocusNow)
        except Exception:
            pass
    else:
        g.es('ignoring: %s' % fn)
#@+node:ekr.20050920084036.169: *5* removeDirectory
def removeDirectory (self,event):

    '''Prompt for the name of a directory and delete it.'''

    k = self.k ; state = k.getState('remove_directory')

    if state == 0:
        k.setLabelBlue('Remove Directory: ')
        k.extendLabel(os.getcwd() + os.sep)
        k.getArg(event,'remove_directory',1,self.removeDirectory)
    else:
        k.keyboardQuit()
        k.clearState()
        try:
            os.rmdir(k.arg)
            k.setStatusLabel('Removed: %s' % k.arg)
        except Exception:
            k.setStatusLabel('Not Removed: %s' % k.arg)
#@+node:ekr.20050920084036.170: *5* saveFile (changed)
def saveFile (self,event):

    '''Prompt for the name of a file and put the body text of the selected node into it..'''

    w = self.editWidget(event)
    if not w: return

    fileName = g.app.gui.runSaveFileDialog(
        initialfile = None,
        title='save-file',
        filetypes = [("Text","*.txt"), ("All files","*")],
        defaultextension = ".txt")

    if not fileName: return

    try:
        f = open(fileName,'w')
        s = w.getAllText()
        if not g.isPython3: # 2010/08/27
            s = g.toEncodedString(s,encoding='utf-8',reportErrors=True)
        f.write(s)
        f.close()
    except IOError:
        g.es('can not create',fileName)
#@+node:ekr.20060205164707: *4* HelpCommandsClass
class HelpCommandsClass (BaseEditCommandsClass):
    '''A class to load files into buffers and save buffers to files.'''
    # pylint: disable=anomalous-backslash-in-string
    @others
#@+node:ekr.20060205165501: *5* getPublicCommands (helpCommands)
def getPublicCommands (self):

    return {
    'help':                             self.help,
    'help-for-abbreviations':           self.helpForAbbreviations,
    'help-for-autocompletion':          self.helpForAutocompletion,
    'help-for-bindings':                self.helpForBindings,
    'help-for-command':                 self.helpForCommand,
    'help-for-creating-external-files': self.helpForCreatingExternalFiles,
    'help-for-debugging-commands':      self.helpForDebuggingCommands,
    'help-for-drag-and-drop':           self.helpForDragAndDrop,
    'help-for-dynamic-abbreviations':   self.helpForDynamicAbbreviations,
    'help-for-find-commands':           self.helpForFindCommands,
    'help-for-minibuffer':              self.helpForMinibuffer,
    'help-for-python':                  self.pythonHelp,
    'help-for-regular-expressions':     self.helpForRegularExpressions,
    'help-for-scripting':               self.helpForScripting,
    'print-settings':                   self.printSettings,
    }
#@+node:ekr.20130412173637.10333: *5* help
def help (self,event=None):

    '''Prints and introduction to Leo's help system.'''

    << define rst_s >>
    self.c.putHelpFor(rst_s)
#@+node:ekr.20130412173637.10330: *6* << define rst_s >> (F1)
@language rest

rst_s = '''

**Welcome to Leo's help system.**

Alt-0 (vr-toggle) hides this help message.

To learn about ``<Alt-X>`` commands, type::
    
    <Alt-X>help-for-minibuffer<Enter>
    
To get a list of help topics, type::
    
    <Alt-X>help-<tab>
    
For Leo commands (tab completion allowed), type::
    
    <Alt-X>help-for-command<Enter>
    <a Leo command name><Enter>
    
To use Python's help system, type::
    
    <Alt-X>help-for-python<Enter>
    <a python symbol><Enter>

'''
#@+node:ekr.20100901080826.5850: *5* helpForAbbreviations
def helpForAbbreviations (self,event=None):
    '''Prints a discussion of abbreviations.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20110530082209.18251: *6* << define s >> (helpForAbbreviations)
@language rest

s = '''\

About Abbreviations
-------------------

Alt-0 (vr-toggle) hides this help message.

Leo optionally expands abbreviations as you type.

Abbreviations typically end with something like ";;" so they won't trigger
by accident.

You define abbreviations in @data abbreviations nodes or @data
global-abbreviations nodes. None come predefined, but leoSettings.leo
contains example abbreviations in the node::

    @@data abbreviations examples

Abbreviations can simply be shortcuts::

    ncn;;=@nocolor
    
Abbreviations can span multiple lines. Continued lines start with \\:, like
this::

    form;;=<form action="main_submit" method="get" accept-charset="utf-8">
    \:<p><input type="submit" value="Continue &rarr;"></p>
    \:</form>\n

Abbreviations can define templates in which <\|a-field-name\|> denotes a field
to be filled in::

    input;;=<input type="text/submit/hidden/button"
    \:name="<|name|>"
    \:value="" id="<|id|>">\n

Typing ",," after inserting a template selects the next field.

Abbreviations can execute **abbreviation scripts**, delimited by {\|{ and
}\|}::

    date;;={|{import time ; x=time.asctime()}|}
    ts;;={|{import time ; x=time.strftime("%Y%m%d%H%M%S")}|}
    
For example, typing ts;; gives::

    20131009171117
    
It's even possible to define a context in which abbreviation scripts execute.

See leoSettings.leo for full details.

'''
#@+node:ekr.20060226131603.1: *5* helpForAutocompletion
def helpForAutocompletion (self,event=None):
    '''Prints a discussion of autocompletion.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20110530082209.18252: *6* << define s >> (helpForAutocompletion)
# @pagewidth 40
@language rest

s = '''

About Autocompletion and Calltips
---------------------------------

Alt-0 (vr-toggle) hides this help message.

This documentation describes both
autocompletion and calltips.

Typing a period when @language python is
in effect starts autocompletion. Typing
'(' during autocompletion shows the
calltip. Typing Return or Control-g
(keyboard-quit) exits autocompletion or
calltips.

Autocompletion
==============

Autocompletion shows what may follow a
period in code. For example, after
typing g. Leo will show a list of all
the global functions in leoGlobals.py.
Autocompletion works much like tab
completion in the minibuffer. Unlike the
minibuffer, the presently selected
completion appears directly in the body
pane.

A leading period brings up 'Autocomplete
Modules'. (The period goes away.) You
can also get any module by typing its
name. If more than 25 items would appear
in the Autocompleter tab, Leo shows only
the valid starting characters. At this
point, typing an exclamation mark shows
the complete list. Thereafter, typing
further exclamation marks toggles
between full and abbreviated modes.

If x is a list 'x.!' shows all its
elements, and if x is a Python
dictionary, 'x.!' shows list(x.keys()).
For example, 'sys.modules.!' Again,
further exclamation marks toggles
between full and abbreviated modes.

During autocompletion, typing a question
mark shows the docstring for the object.
For example: 'g.app?' shows the
docstring for g.app. This doesn't work
(yet) directly for Python globals, but
'__builtin__.f?' does. Example:
'__builtin__.pow?' shows the docstring
for pow.

Autocompletion works in the Find tab;
you can use <Tab> to cycle through the
choices. The 'Completion' tab appears
while you are doing this; the Find tab
reappears once the completion is
finished.

Calltips
========

Calltips appear after you type an open
parenthesis in code. Calltips shows the
expected arguments to a function or
method. Calltips work for any Python
function or method, including Python's
global function. Examples:

a) g.toUnicode(
   gives:
   g.toUnicode(s,encoding, reportErrors=False

b) c.widgetWantsFocusNow
   gives:
   c.widgetWantsFocusNow(w

c) reduce(
   gives:
   reduce(function, sequence[,initial]) -> value

The calltips appear directly in the text
and the argument list is highlighted so
you can just type to replace it. The
calltips appear also in the status line
for reference after you have started to
replace the args.

Options
=======

Both autocompletion and calltips are
initially enabled or disabled by the
enable_autocompleter_initially and
enable_calltips_initially settings in
leoSettings.leo. You may enable or
disable these features at any time with
these commands: enable-autocompleter,
enable-calltips, disable-autocompleter
and disable-calltips. '''
#@+node:ekr.20060205170335: *5* helpForBindings
def helpForBindings (self,event=None):
    '''Prints a discussion of keyboard bindings.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20110530082209.18253: *6* << define s >> (helpForBindings)
# @pagewidth 40
@language rest

s = '''

About Key Bindings
------------------

Alt-0 (vr-toggle) hides this help message.

A shortcut specification has the form:

command-name = shortcutSpecifier

or

command-name ! pane = shortcutSpecifier

The first form creates a binding for all
panes except the minibuffer. The second
form creates a binding for one or more
panes. The possible values for 'pane'
are:

====    ===============
pane    bound panes
====    ===============
all     body,log,tree
body    body
log     log
mini    minibuffer
text    body,log
tree    tree
====    ===============

You may use None as the specifier.
Otherwise, a shortcut specifier consists
of a head followed by a tail. The head
may be empty, or may be a concatenation
of the following: (All entries in each
row are equivalent)::

    Shift+ Shift-
    Alt+ or Alt-
    Control+, Control-, Ctrl+ or Ctrl-

Notes:

1. The case of plain letters is significant:
   a is not A.

2. The Shift- (or Shift+) prefix can be
   applied *only* to letters or
   multi-letter tails. Leo will ignore
   (with a warning) the shift prefix
   applied to other single letters,
   e.g., Ctrl-Shift-(

3. The case of letters prefixed by
   Ctrl-, Alt-, Key- or Shift- is *not*
   significant.

The following table illustrates these
rules. In each row, the first entry is
the key (for k.bindingsDict) and the
other entries are equivalents that the
user may specify in leoSettings.leo::

    a, Key-a, Key-A
    A, Shift-A
    Alt-a, Alt-A
    Alt-A, Alt-Shift-a, Alt-Shift-A
    Ctrl-a, Ctrl-A
    Ctrl-A, Ctrl-Shift-a, Ctrl-Shift-A
    !, Key-!,Key-exclam,exclam

'''
#@+node:ekr.20060417203717: *5* helpForCommand & helpers
def helpForCommand (self,event):
    '''Prompts for a command name and prints the help message for that command.'''
    c,k = self.c,self.k
    s = '''\
Alt-0 (vr-toggle) hides this help message.

Type the name of the command, followed by Return.
'''
    c.putHelpFor(s)
    c.minibufferWantsFocusNow()
    k.fullCommand(event,help=True,helpHandler=self.helpForCommandFinisher)
#@+node:ekr.20120521114035.9870: *6* getBindingsForCommand
def getBindingsForCommand(self,commandName):

    c = self.c ; k = c.k
    data = [] ; n1 = 4 ; n2 = 20
    d = k.bindingsDict
    for stroke in sorted(d):
        assert g.isStroke(stroke),repr(stroke)
        aList = d.get(stroke,[])
        for si in aList:
            assert g.isShortcutInfo(si),si
            if si.commandName == commandName:
                pane = '' if si.pane=='all' else ' %s:' % (si.pane)
                s1 = pane
                s2 = k.prettyPrintKey(stroke)
                s3 = si.commandName
                n1 = max(n1,len(s1))
                n2 = max(n2,len(s2))
                data.append((s1,s2,s3),)

    data.sort(key=lambda x: x[1])
    return ','.join(['%s %s' % (s1,s2) for s1,s2,s3 in data]).strip()
#@+node:ekr.20120521114035.9871: *6* helpForCommandFinisher
def helpForCommandFinisher (self,commandName):

    c,s = self.c,None
    if commandName and commandName.startswith('help-for-'):
        # Execute the command itself.
        c.k.simulateCommand(commandName)
    else:
        if commandName:
            bindings = self.getBindingsForCommand(commandName)
            func = c.commandsDict.get(commandName)
            s = g.getDocStringForFunction(func)
            if s:
                s = self.replaceBindingPatterns(s)
            else:
                s = 'no docstring available'
            # Create the title.
            s2 = '%s (%s)' % (commandName,bindings) if bindings else commandName
            underline = '+' * len(s2)
            # title = '%s\n%s\n%s\n\n' % (underline,s2,underline)
            title = '%s\n%s\n\n' % (s2,underline)
            # Fixes bug 618570:
            s = title + ''.join([
                line.lstrip() if line.strip() else '\n'
                    for line in g.splitLines(s)])
        else:
            << set s to about help-for-command >>
        c.putHelpFor(s) # calls g.adjustTripleString.
#@+node:ekr.20120521114035.9872: *7* << set s to about help-for-command >>
s = '''\

++++++++++++++++++++++++
About Leo's help command
++++++++++++++++++++++++

Invoke Leo's help-for-command as follows::

    <F1>
    <Alt-X>help-for-command<return>

Next, type the name of one of Leo's commands.
You can use tab completion.  Examples::

    <F1><tab>           shows all commands.
    <F1>help-for<tab>   shows all help-for- commands.

Here are the help-for commands::

    help-for-abbreviations
    help-for-autocompletion
    help-for-bindings
    help-for-command
    help-for-debugging-commands
    help-for-dynamic-abbreviations
    help-for-find-commands
    help-for-minibuffer
    help-for-python
    help-for-regular-expressions

'''
#@+node:ekr.20120524151127.9886: *6* replaceBindingPatterns
def replaceBindingPatterns (self,s):

    '''For each instance of the pattern !<command-name>! is s,
    replace the pattern by the key binding for command-name.'''

    c = self.c
    pattern = re.compile('!<(.*)>!')
    while True:
        m = pattern.search(s,0)
        if m is None: break
        name = m.group(1)
        junk,aList = c.config.getShortcut(name)
        for si in aList:
            if si.pane == 'all':
                key = c.k.prettyPrintKey(si.stroke.s)
                break
        else: key = '<Alt-X>%s<Return>' % name
        s = s[:m.start()] + key + s[m.end():]
    return s

#@+node:ekr.20131213163822.16472: *5* helpForCreatingExternalFiles
def helpForCreatingExternalFiles(self,event=None):
    '''Prints a discussion of creating external files.'''
    << define s >>
    s = s.replace('\\','')
    self.c.putHelpFor(s)
#@+node:ekr.20131213163822.16477: *6* << define s >> (helpForCreatingExternalFiles)
@language rest

s = '''

Creating External Files
-------------------------

This help discusses only @file nodes.
For other ways of creating external files, see::

    http://leoeditor.com/tutorial-programming.html or
    http://leoeditor.com/directives.html

Leo creates external files in an unusual way.
Please fee free to ask for help::

    https://groups.google.com/forum/#!forum/leo-editor or
    http://webchat.freenode.net/?channels=%23leo&uio=d4

Overview
========

Leo creates **external files** (files on your file system) from
**@file nodes** and *all the descendants* of the @file node.
Examples::

    @file spam.py
    @file ../foo.c
    @file ~/bar.py
    
A single Leo outline may contain many @file nodes. As a result, Leo
outlines function much like project files in other IDE's (Integrated
development environments).

Within an @file tree, simple text markup (discussed next) tells Leo how
to create the external file from the @file node and its descendants.

Markup
======
   
Section references and the \\@all and \\@others directives tell
Leo how to create external files.

- A **section name** is any text of the form::

    <\\< any text >\\> (>> must not appear in "any text".)

- A **section definition node** is any node whose headline starts with a section name.

- A **section reference** is a section name that appears in body text.

Leo creates external files containing \\@others directives by writing the
*expansion* of the @file node. The **expansion** of *any* node is the
node's body text after making these text **substitutions**:

- Leo replaces \\@all by the *unexpanded* body text of *all* nodes.

- Leo replaces \\@others with the *expansion* of all descendant nodes
  **except** section definition nodes. That's how \\@others got its name.

- Leo replaces section references by the *expansion* of the body text of
  the corresponding section definition node.
   
Whitespace is significant before \\@others and section references. Leo adds
the leading whitespace appearing before each \\@others directive or section
reference to every line of their expansion. As a result, Leo can generate
external files even for Python.  The following cute trick works::

    if 1:
        <\\< a section >\\>
    if 0:
        \\@others

**Notes**:

- Any node may contain a single \\@others directive. No node may contain more
  than one \@others directive.

- Nodes that *aren't* section definition nodes are included in the expansion
  of the *nearest* ancestor node containing an @others directive.

**Example 1**: The body of the @file node for a typical Python module will
look something like::

    '\\''A docstring.'\\''
    <\\< imports >\\>
    \@others
    if __name__ == '__main__':
        main()
        
**Example 2**:  Here is a typical Python class definition in Leo::

    class MyClass:
        '\\''A docstring.'\\''
        \@others
        
\@first and @last
=================

The @first directive forces lines to appear before the first sentinel of a
external file. For example::

    @first #! /usr/bin/env python
    @first # -*- coding: utf-8 -*-

Similarly, @last forces lines to appear after the last sentinel.

\\@path
=======

Rather than specifying long paths in @file nodes, you can specify a path in
an ancestor @path node.

For example, suppose three nodes have the following headlines::

    @path a
        @path b
            @file c/d.py

The @file node creates the file a/b/c/d.py

Within @path and @<file> paths, {{exp}} gets evaluated with the following
predefined symbols: c, g, p, os and sys.  For example::

    @file {{os.path.abspath(os.curdir)}}/abc.py

'''
#@+node:ekr.20070501092655: *5* helpForDebuggingCommands
def helpForDebuggingCommands (self,event=None):
    '''Prints a discussion of of Leo's debugging commands.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20070501092655.1: *6* << define s >> (helpForDebuggingCommands)
# @pagewidth 40
@language rest

s = '''

About Debugging Commands
------------------------

Alt-0 (vr-toggle) hides this help message.

The following commands are useful for debugging::

    collect-garbage:   Invoke the garbage collector.
    debug:             Start an external debugger in another process.
    disable-gc-trace:  Disable tracing of the garbage collector.
    dump-all-objects:  Print a summary of all existing Python objects.
    dump-new-objects:  Print a summary of all newly-created Python objects.
    enable-gc-trace:   Enable tracing of the garbage collector.
    free-tree-widgets: Free all widgets used in Leo's outline pane.
    print-focus:       Print information about the requested focus.
    print-stats:       Print statistics about existing Python objects.
    print-gc-summary:  Print a brief summary of all Python objects.
    run-unit-tests:    Run unit tests in the presently selected tree.
    verbose-dump-objects: Print a more verbose listing of all existing Python objects.

Leo also has many debugging settings that enable and disable traces.
For details, see the node: @settings-->Debugging in leoSettings.leo.
'''
#@+node:ekr.20141008000933.20113: *5* helpForDragAndDrop
def helpForDragAndDrop (self,event=None):
    '''Prints a discussion of of Leo's debugging commands.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20141008000933.20114: *6* << define s >> (helpForDragAndDrop
# @pagewidth 40
@language rest

s = '''

About Drag and Drop
===================

Nodes
-----

You may drag nodes from within a Leo outline or between Leo outlines,
including nodes from separately running copies of Leo.

To drag, click and hold on a headline and drag it over another headline.
Control-drags create clones; regular drags move the node.


Files
-----

You may also drag one or more files from a file folder over a headline.

For text files, Leo will create @auto, @edit or @file nodes as appropriate.

Dragging .leo files from a file folder to a Leo outline works like the
open-outline command. Control-dragging .leo files copies all nodes of the
.leo file to the receiving outline, under a single node called::

    From <name of .leo file>

'''
#@+node:ekr.20131029061413.17094: *5* helpForDynamicAbbreviations
def helpForDynamicAbbreviations (self,event=None):
    '''Prints a discussion of abbreviations.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20131029061413.17095: *6* << define s >> (helpForDynamicAbbreviations)
@language rest

s = '''

About Dynamic Abbreviations
---------------------------

Alt-0 (vr-toggle) hides this help message.

.. Description taken from http://www.emacswiki.org/emacs/DynamicAbbreviations

A dynamic abbreviation (dabbrev) is like a normal abbreviation except:

- You do not have to define it(!)
- You expand it with Alt-/ (dabbrev-expand) or Alt-Ctrl-/ (dabbrev-completion)

For example, suppose the text aLongIvarName appears anywhere in the
outline. To type this name again type::

    aLong<Alt-/>

You will see a list of possible completions in the log pane.

Alt-Ctrl-/ (dabbrev-completion) inserts the longest prefix of all
completions immediately.  For instance, suppose the following appear in text::

    aVeryLongIvarName
    aVeryLongMethodName
    
Typing::

    aVery<Alt-Ctrl-/>
    
will immediately extend the typing to::

    aVeryLong

'''
#@+node:ekr.20060205170335.1: *5* helpForFindCommands
def helpForFindCommands (self, event=None):
    '''Prints a discussion of of Leo's find commands.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20130411023826.16595: *6* << define s >> (help-for-find-commands)
@language rest

s = '''

Finding & replacing text
------------------------

Alt-0 (vr-toggle) hides this help message.

**Ctrl-F** (start-search) shows the Find pane
and puts focus in the find box.

Enter the find text and the replacement text if desired::

    Tab switches focus from widget to widget.
    Return executes the find-next command.

When Leo selects the found text you can do::

    Ctrl-equal (replace)
    Ctrl-minus (replace-then-find)
    F3 (find-next)
    F2 (find-previous)
    Ctrl-G (keyboard-quit)
    anything else :-)

You can Leo's commands toggle check boxes and radio buttons.
These commands are listed in the Search menu.

You can execute these commands (and see their key bindings)
using the minibuffer::

    <Alt-X>tog<tab>f<tab>   or
    <Alt-X>set<tab>f<tab>

Incremental searching
---------------------

Incremental search is done only from the minibuffer::

    Alt-I (isearch forward)
    Alt-R (isearch backward)
    BackSpace retracts the search
    All other characters extend the search

During an incremental search::

    Enter or Ctrl-G stops the search.
    Alt-S finds the search string again.
    Alt-R ditto for reverse searches.
'''
#@+node:ekr.20051014170754: *5* helpForMinibuffer
def helpForMinibuffer (self,event=None):
    '''Print a messages telling you how to get started with Leo.'''
    # A bug in Leo: triple quotes puts indentation before each line.
    c = self.c    
    << define s >>
    c.putHelpFor(s)
#@+node:ekr.20120522024827.9899: *6* << define s >> (helpForMinibuffer)
@language rest

s = '''\

About the Minibuffer
--------------------

Alt-0 (vr-toggle) hides this help message.

The mini-buffer is intended to be like the Emacs buffer:

full-command: (default shortcut: Alt-x) Puts the focus in the minibuffer. Type a
full command name, then hit <Return> to execute the command. Tab completion
works, but not yet for file names.

quick-command-mode (default shortcut: Alt-x). Like Emacs Control-C. This mode is
defined in leoSettings.leo. It is useful for commonly-used commands.

universal-argument (default shortcut: Alt-u). Like Emacs Ctrl-u. Adds a repeat
count for later command. Ctrl-u 999 a adds 999 a's. Many features remain
unfinished.

keyboard-quit (default shortcut: Ctrl-g) Exits any minibuffer mode and puts
the focus in the body pane.

Use the help-for-command command to see documentation for a particular command.
'''
#@+node:ekr.20120522024827.9897: *5* helpForRegularExpressions
def helpForRegularExpressions (self, event=None):
    '''Prints a discussion of of Leo's find commands.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20120522024827.9898: *6* << define s >> (helpForRegularExpressions)
@language rest

# Using raw string is essential.

s = r'''

About regular expressions
-------------------------

Alt-0 (vr-toggle) hides this help message.

Python's regular expressions, http://docs.python.org/library/re.html,
are valid in find patterns::

    .               Matches any character (including newline if DOTALL flag specified).
    ^               Matches start of the string (of every line in MULTILINE mode).
    $               Matches end of the string (of every line in MULTILINE mode).
    *               0 or more of preceding regular expression (as many as possible).
    +               1 or more of preceding regular expression (as many as possible).
    ?               0 or 1 occurrence of preceding regular expression.
    *?, +?, ??      Same as *, + and ? but matches as few characters as possible.
    {m,n}           Matches from m to n repetitions of preceding RE.
    {m,n}?          Same as {m,n}, but attempting to match as few repetitions as possible.
    [ ]             Defines character set: e.g. '[a-zA-Z]' to match all letters (see also \w \S).
    [^ ]            Defines complemented character set: matches if char is NOT in set.
    \               Escapes special chars '*?+&$|()' and introduces special sequences (see below).
                    If not using a raw string, write as '\\' in the pattern string.
    \\              Matches a literal '\'.
    |               Specifies alternative: 'foo|bar' matches 'foo' or 'bar'.
    (...)           Matches any RE inside (), and delimits a group.
    (?:...)         Mathces RE inside (), but doesn't delimit a group.
    (?P<name>...)   Matches any RE inside (), and delimits a named group.
                    r'(?P<id>[a-zA-Z_]\w*)' defines a group named id.
    (?P=name)       Matches whatever text was matched by the earlier group named name.
    (?=...)         Matches if ... matches next, but doesn't consume any of the string
                    'Isaac (?=Asimov)' matches 'Isaac' only if followed by 'Asimov'.
    (?!...)         Matches if ... doesn't match next. Negative of (?=...).
    (?<=...)        Matches if the current position in the string is preceded by a match
                    for ... that ends at the current position.
                    This is called a positive lookbehind assertion.
    (?<!...)        Matches if the current position in the string is not preceded by a match for ...
                    This is called a negative lookbehind assertion.
    (?(group)A|B)   Group is either a numeric group ID or a group name defined with (?Pgroup...)
                    earlier in the expression.
                    If the specified group matched, the regular expression pattern A will be tested
                    against the string; if the group didn't match, the pattern B will be used instead.
    (?#...)         A comment; ignored.
    (?letters)      Each letter is in 'ilmsux' and sets the corresponding flag.
                    re.I, re.L, re.M, re.S, re.U, re.X.
    \number         Matches content of the group of the same number.     
    \A              Matches only at the start of the string.
    \b              Empty str at beginning or end of word:
                    '\bis\b' matches 'is', but not 'his'.
    \B              Empty str NOT at beginning or end of word.
    \d              Any decimal digit:          [0-9]
    \D              Any non-decimal digit char  [^0-9]).
    \s              Any whitespace char         [ \t\n\r\f\v]
    \S              Any non-whitespace char     [^ \t\n\r\f\v]
    \w              Any alphaNumeric char (depends on LOCALE flag).
    \W              Any non-alphaNumeric char (depends on LOCALE flag).
    \Z              Matches only at the end of the string.

'''
#@+node:ekr.20131213163822.16473: *5* helpForScripting
def helpForScripting(self,event=None):
    '''Prints a discussion of Leo scripting.'''
    << define s >>
    self.c.putHelpFor(s)
#@+node:ekr.20131213163822.16475: *6* << define s >> (helpForScripting)
@language rest

s = '''

Summary of Leo Scripting
-------------------------

Overview
========

Any Leo node may contain a Python script.

Ctrl-B (execute-script) executes the body text of the presently selected node.

execute-script creates the script using @others and section references:
**you can create complex scripts from a node and its descendants.**

As discussed below, execute-script predefines three variables: c, g and p.
Using these variables, scripts may easily do any of the following:

- Gain access to all data contained in any Leo outline.
- Traverse the data in any outline.
- Use utility classes and function in the leo.core.leoGlobals module.
- Execute any code in Leo's own code base.

*Tip*: use Alt-1 (toggle-autocompleter) and Alt-2 (toggle-calltips) as aids to memory and to speed typing.

Predefined symbols
==================

The execute-script command predefines three variables::

    c: The commander of the present outline.
    g: The leo.core.leoGlobals module.
    p: The presently selected position, c.p.
    
@test scripts predefine all the above, plus::

    self: The instance of unittest.TestCase
    
Commands class
==============
    
A commander is an instance of the Commands class in leo.core.leoCommands.
A commander represents all outline data and most of Leo's source code.
Here are the most important ivars of the Commands class::

    c.frame         c's outer frame, a LeoFrame instance.
    c.user_dict     a temporary dict for use of scripts and plugins.
    c.redraw()
    c.positionExists(p)

Here is a partial list of the **official ivars** of any commander c:

    c.frame                 The frame containing the log,body,tree, etc.
    c.frame.body            The body pane.
    c.frame.body.widget     The gui widget for the body pane.
    c.frame.body.wrapper    The high level interface for the body widget.
    c.frame.iconBar         The icon bar.
    c.frame.log             The log pane.
    c.frame.log.widget      The gui widget for the log pane.
    c.frame.log.wrapper     The high-level inteface for the log pane.
    c.frame.tree            The tree pane.
    
VNode class
===========

All data in Leo outlines resides in vnodes.
All clones of the same node share the same VNode.
Here are the most important ivars and properties of the VNode class::

    v.b: v's body text.
    v.h: v's headline text.
    v.u: v.unknownAttributes, a persistent Python dictionary.
    
v.u (uA's or unknownAttributes or userAttributes) allow plugins or scripts
to associate persistent data with vnodes. For details see the section about
userAttributes in the Customizing Leo chapter.

position class
==============

A position represents the state of a traversal of an outline.
Because of clones, the same VNode may appear multiple times during a traversal.

Properties of the position class::

    p.b: same as p.v.b.
    p.h: same as p.v.h.
    p.u: same as p.v.u.

Getter methods of the position class::

    p.back()
    p.children()
    p.firstChild()
    p.hasBack()
    p.hasChildren()
    p.hasNext()
    p.hasParent()
    p.hasThreadBack()
    p.hasThreadNext()
    p.isAncestorOf(p2)
    p.isAnyAtFileNode()
    p.isAt...Node()
    p.isCloned()
    p.isDirty()
    p.isExpanded()
    p.isMarked()
    p.isRoot()
    p.isVisible()
    p.lastChild()
    p.level()
    p.next()
    p.nodeAfterTree()
    p.nthChild()
    p.numberOfChildren()
    p.parent()
    p.parents()
    p.threadBack()
    p.threadNext()
    p.visBack()
    p.visNext()
    
Setter methods of the position class::

    p.setDirty()
    p.setMarked()

Methods that operate on nodes::

    p.clone()
    p.contract()
    p.doDelete(new_position)
    p.expand()
    p.insertAfter()
    p.insertAsNthChild(n)
    p.insertBefore()
    p.moveAfter(p2)
    p.moveToFirstChildOf(parent,n)
    p.moveToLastChildOf(parent,n)
    p.moveToNthChildOf(parent,n)
    p.moveToRoot(oldRoot=None)
        # oldRoot **must** be the old root position if it exists.

The following position methods move positions *themselves*: they change the
node to which a position refers. They do *not* change outline structure in
any way! Use these when generators are not flexible enough::

    p.moveToBack()
    p.moveToFirstChild()
    p.moveToLastChild()
    p.moveToLastNode()
    p.moveToNext()
    p.moveToNodeAfterTree(p2)
    p.moveToNthChild(n)) 
    p.moveToParent()
    p.moveToThreadBack()
    p.moveToThreadNext()
    p.moveToVisBack(c)
    p.moveToVisNext(c)

Generators
==========

The following Python generators return positions::

    c.all_positions()
    c.all_unique_positions()
    p.children()
    p.parents()
    p.self_and_parents()
    p.self_and_siblings()
    p.following_siblings()
    p.subtree()
    p.self_and_subtree()
    
The leo.core.leoGlobals module
==============================

**g vars**::

    g.app
    g.app.gui
    g.app.windowlist
    g.unitTesting
    g.user_dict  # a temporary dict for use of scripts and plugins.
    
**g decorator**::

    @g.command(command-name)
    
**g functions** (the most interesting: there are many more in leoGlobals.py)::
    
    g.angleBrackets()
    g.app.commanders()
    g.app.gui.guiName()
    g.es(*args,**keys)
    g.es_print(*args,**keys)
    g.es_exception()
    g.getScript(c,p,
        useSelectedText=True,
        forcePythonSentinels=True,
        useSentinels=True)
    g.openWithFileName(fileName,old_c=None,gui=None)
    g.os_path_... # Wrappers for os.path methods.
    g.pdb(message='')
    g.toEncodedString(s,encoding='utf-8',reportErrors=False)
    g.toUnicode(s, encoding='utf-8',reportErrors=False)
    g.trace(*args,**keys)
    g.warning(*args,**keys)

'''
#@+node:ekr.20070418074444: *5* printSettings
def printSettings (self,event=None):
    '''
    Prints the value of every setting, except key bindings and commands and
    open-with tables. The following shows where the active setting came
    from:

    -     leoSettings.leo,
    - [D] default settings.
    - [F] indicates the file being loaded,
    - [M] myLeoSettings.leo,
    '''
    self.c.config.printSettings()
#@+node:ekr.20060602154458: *5* pythonHelp
def pythonHelp (self,event=None):
    '''Prompt for a arg for Python's help function, and put it to the log pane.'''
    c = self.c ; k = c.k ; tag = 'python-help' ; state = k.getState(tag)
    if state == 0:
        c.minibufferWantsFocus()
        k.setLabelBlue('Python help: ')
        k.getArg(event,tag,1,self.pythonHelp)
    else:
        k.clearState()
        k.resetLabel()
        s = k.arg.strip()
        if s:
            # Capture the output of Python's help command.
            old = sys.stdout
            try:
                sys.stdout = stdout = g.FileLikeObject()
                help(str(s))
                s2 = stdout.read()
            finally:
                sys.stdout = old
            # Send it to the vr pane as a <pre> block
            s2 = '<pre>' + s2 + '</pre>'
            c.putHelpFor(s2)
#@+node:ekr.20050920084036.171: *4* KeyHandlerCommandsClass (add docstrings)
class KeyHandlerCommandsClass (BaseEditCommandsClass):

    '''User commands to access the keyHandler class.'''

    @others
#@+node:ekr.20050920084036.172: *5*  ctor (KeyHandlerCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.
#@+node:ekr.20050920084036.173: *5* getPublicCommands (KeyHandlerCommandsClass)
def getPublicCommands (self):

    k = self.k

    if k:
        return {
            'auto-complete':            k.autoCompleter.autoComplete,
            'auto-complete-force':      k.autoCompleter.autoCompleteForce,
            'digit-argument':           k.digitArgument,
            'disable-autocompleter':    k.autoCompleter.disableAutocompleter,
            'disable-calltips':         k.autoCompleter.disableCalltips,
            'enable-autocompleter':     k.autoCompleter.enableAutocompleter,
            'enable-calltips':          k.autoCompleter.enableCalltips,
            'exit-named-mode':          k.exitNamedMode,
            'full-command':             k.fullCommand, # For menu.
            # 'hide-mini-buffer':         k.hideMinibuffer,
            'menu-shortcut':            self.menuShortcutPlaceHolder,
            'mode-help':                k.modeHelp,
            'negative-argument':        k.negativeArgument,
            'number-command':           k.numberCommand,
            'number-command-0':         k.numberCommand0,
            'number-command-1':         k.numberCommand1,
            'number-command-2':         k.numberCommand2,
            'number-command-3':         k.numberCommand3,
            'number-command-4':         k.numberCommand4,
            'number-command-5':         k.numberCommand5,
            'number-command-6':         k.numberCommand6,
            'number-command-7':         k.numberCommand7,
            'number-command-8':         k.numberCommand8,
            'number-command-9':         k.numberCommand9,
            'print-bindings':           k.printBindings,
            'print-buttons':            k.printButtons,
            'print-commands':           k.printCommands,
            'repeat-complex-command':   k.repeatComplexCommand,
            # 'scan-for-autocompleter':   k.autoCompleter.scan,
            'set-command-state':        k.setCommandState,
            'set-insert-state':         k.setInsertState,
            'set-overwrite-state':      k.setOverwriteState,
            'show-calltips':            k.autoCompleter.showCalltips,
            'show-calltips-force':      k.autoCompleter.showCalltipsForce,
            # 'show-mini-buffer':         k.showMinibuffer,
            'toggle-autocompleter':     k.autoCompleter.toggleAutocompleter,
            'toggle-calltips':          k.autoCompleter.toggleCalltips,
            #'toggle-mini-buffer':       k.toggleMinibuffer,
            'toggle-input-state':       k.toggleInputState,
            'universal-argument':       k.universalArgument,
        }
    else:
        return {}
#@+node:ekr.20131221055224.17570: *5* def menuShortcutPlaceHolder
g.command('menu-shortcut')
def menuShortcutPlaceHolder(self,event=None):
    '''This will never be called.
    It serves as a placeholder for the print-bindings command.'''
#@+node:ekr.20050920084036.174: *4* KillBufferCommandsClass
class KillBufferCommandsClass (BaseEditCommandsClass):

    '''A class to manage the kill buffer.'''

    @others
#@+node:ekr.20050920084036.175: *5*  ctor & finishCreate (KillBufferCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.addWsToKillRing = c.config.getBool('add-ws-to-kill-ring')
    self.k = None
    self.kbiterator = self.iterateKillBuffer()
    self.last_clipboard = None  # For interacting with system clipboard.
    self.lastYankP = None
        # Position of the last item returned by iterateKillBuffer.
    self.reset = None
        # The index of the next item to be returned in
        # g.app.globalKillBuffer by iterateKillBuffer.

def finishCreate (self):

    BaseEditCommandsClass.finishCreate(self)
        # Call the base finishCreate.
        # This sets self.k
#@+node:ekr.20050920084036.176: *5*  getPublicCommands
def getPublicCommands (self):

    return {
        'backward-kill-sentence':   self.backwardKillSentence,
        'backward-kill-word':       self.backwardKillWord,
        'clear-kill-ring':          self.clearKillRing,
        'kill-line':                self.killLine,
        'kill-to-end-of-line':      self.killToEndOfLine,
        'kill-word':                self.killWord,
        'kill-sentence':            self.killSentence,
        'kill-region':              self.killRegion,
        'kill-region-save':         self.killRegionSave,
        'kill-ws':                  self.killWs,
        'yank':                     self.yank,
        'yank-pop':                 self.yankPop,
        'zap-to-character':         self.zapToCharacter,
    }
#@+node:ekr.20050920084036.183: *5* addToKillBuffer
def addToKillBuffer (self,text):

    '''Insert the text into the kill buffer if force is True or
    the text contains something other than whitespace.'''

    if self.addWsToKillRing or text.strip():
        g.app.globalKillBuffer = [
            z for z in g.app.globalKillBuffer if z != text]
        g.app.globalKillBuffer.insert(0,text)
#@+node:ekr.20050920084036.181: *5* backwardKillSentence
def backwardKillSentence (self,event):

    '''Kill the previous sentence.'''

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    ins = w.getInsertPoint()
    i = s.rfind('.',ins)
    if i == -1: return

    undoType='backward-kill-sentence'

    self.beginCommand(undoType=undoType)

    i2 = s.rfind('.',0,i) + 1
    self.kill(event,i2,i+1,undoType=undoType)
    self.c.frame.body.forceFullRecolor()
    w.setInsertPoint(i2)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.180: *5* backwardKillWord & killWord
def backwardKillWord (self,event):
    '''Kill the previous word.'''
    c = self.c ; e = c.editCommands
    self.beginCommand(undoType='backward-kill-word')
    e.backwardWord(event)
    self.killWordHelper(event,'back')

def killWord (self,event):
    '''Kill the word containing the cursor.'''
    self.beginCommand(undoType='kill-word')
    self.killWordHelper(event,'forward')

def killWordHelper(self,event,direction):
    c = self.c ; e = c.editCommands ; w = e.editWidget(event)
    # self.killWs(event)
    e.extendToWord(event,direction)
    i,j = w.getSelectionRange()
    self.kill(event,i,j,undoType = None)
    c.frame.body.forceFullRecolor()
    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20051216151811: *5* clearKillRing
def clearKillRing (self,event=None):

    '''Clear the kill ring.'''

    g.app.globalKillbuffer = []
#@+node:ekr.20050920084036.185: *5* getClipboard
def getClipboard (self):

    '''Return the contents of the clipboard.'''

    try:
        ctxt = g.app.gui.getTextFromClipboard()
        if not g.app.globalKillBuffer or ctxt != self.last_clipboard:
            self.last_clipboard = ctxt
            if not g.app.globalKillBuffer or g.app.globalKillBuffer [0] != ctxt:
                return ctxt
    except Exception:
        g.es_exception()

    return None
#@+node:ekr.20050920084036.184: *5* iterateKillBuffer
class KillBufferIterClass:

    """Returns a list of positions in a subtree, possibly including the root of the subtree."""

    @others

def iterateKillBuffer (self):

    return self.KillBufferIterClass(self.c)
#@+node:ekr.20071003160252.1: *6* __init__ & __iter__ (iterateKillBuffer)
def __init__(self,c):

    # g.trace('iterateKillBuffer.__init')
    self.c = c
    self.index = 0 # The index of the next item to be returned.

def __iter__(self):

    return self
#@+node:ekr.20071003160252.2: *6* next
def next(self):

    commands = self.c.killBufferCommands
    aList = g.app.globalKillBuffer # commands.killBuffer

    # g.trace(g.listToString([repr(z) for z in aList]))

    if not aList:
        self.index = 0
        return None

    if commands.reset is None:
        i = self.index
    else:
        i = commands.reset
        commands.reset = None

    if i < 0 or i >= len(aList): i = 0
    # g.trace(i)
    val = aList[i]
    self.index = i + 1
    return val

__next__ = next
#@+node:ekr.20050920084036.178: *5* kill
def kill (self,event,frm,to,undoType=None):

    '''A helper method for all kill commands.'''

    w = self.editWidget(event)
    if not w: return
    s = w.get(frm,to)
    # g.trace(repr(s))
    if undoType: self.beginCommand(undoType=undoType)
    self.addToKillBuffer(s)
    g.app.gui.replaceClipboardWith(s)
    w.delete(frm,to)
    w.setInsertPoint(frm)
    if undoType:
        self.c.frame.body.forceFullRecolor()
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20130918042415.11297: *5* killToEndOfLine (New in Leo 4.11)
def killToEndOfLine (self,event):
    '''Kill from the cursor to end of the line.'''
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    # g.trace(ins,j,repr(s[i:j]))
    if ins >= len(s) and g.match(s,j-1,'\n'):
        # Kill the trailing newline of the body text.
        i = max(0,len(s)-1)
        j = len(s)
    elif ins + 1 < j and s[ins:j-1].strip() and g.match(s,j-1,'\n'):
        # Kill the line, but not the newline.
        i,j = ins,j-1
    elif g.match(s,j-1,'\n'):
        i = ins # Kill the newline in the present line.
    else:
        i = j
    if i < j:
        self.kill(event,i,j,undoType='kill-line')
#@+node:ekr.20071003183657: *5* KillLine
def killLine (self,event):
    '''Kill the line containing the cursor.'''
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    i,j = g.getLine(s,ins)
    if ins >= len(s) and g.match(s,j-1,'\n'):
        # Kill the trailing newline of the body text.
        i = max(0,len(s)-1)
        j = len(s)
    elif j > i+1 and g.match(s,j-1,'\n'):
        # Kill the line, but not the newline.
        j -= 1
    else:
        pass # Kill the newline in the present line.
    self.kill(event,i,j,undoType='kill-line')
#@+node:ekr.20050920084036.182: *5* killRegion & killRegionSave & helper
def killRegion (self,event):
    '''Kill the text selection.'''
    self.killRegionHelper(event,deleteFlag=True)

def killRegionSave (self,event):
    '''Add the selected text to the kill ring, but do not delete it.'''
    self.killRegionHelper(event,deleteFlag=False)

def killRegionHelper (self,event,deleteFlag):

    w = self.editWidget(event)
    if not w: return
    i,j = w.getSelectionRange()
    if i == j: return
    s = w.getSelectedText()
    if deleteFlag:
        self.beginCommand(undoType='kill-region')
        w.delete(i,j)
        self.c.frame.body.forceFullRecolor()
        self.endCommand(changed=True,setLabel=True)
    self.addToKillBuffer(s)
    g.app.gui.replaceClipboardWith(s)
    # self.removeRKeys(w)
#@+node:ekr.20050930095323.1: *5* killSentence
def killSentence (self,event):

    '''Kill the sentence containing the cursor.'''

    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    ins = w.getInsertPoint()
    i = s.find('.',ins)
    if i == -1: return

    undoType='kill-sentence'

    self.beginCommand(undoType=undoType)

    i2 = s.rfind('.',0,ins) + 1
    self.kill(event,i2,i+1,undoType=undoType)
    self.c.frame.body.forceFullRecolor()
    w.setInsertPoint(i2)

    self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050930100733: *5* killWs
def killWs (self,event,undoType='kill-ws'):

    '''Kill whitespace.'''

    ws = ''
    w = self.editWidget(event)
    if not w: return
    s = w.getAllText()
    i = j = ins = w.getInsertPoint()

    while i >= 0 and s[i] in (' ','\t'):
        i-= 1
    if i < ins: i += 1

    while j < len(s) and s[j] in (' ','\t'):
        j += 1

    if j > i:
        ws = s[i:j]
        # g.trace(i,j,repr(ws))
        w.delete(i,j)
        if undoType: self.beginCommand(undoType=undoType)
        if self.addWsToKillRing:
            self.addToKillBuffer(ws)
        if undoType: self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050930091642.1: *5* yank
def yank (self,event,pop=False):

    '''yank: insert the first entry of the kill ring.
    yank-pop: insert the next entry of the kill ring.
    '''

    c = self.c ; w = self.editWidget(event)
    if not w: return
    current = c.p
    if not current: return
    text = w.getAllText()
    i, j = w.getSelectionRange()
    clip_text = self.getClipboard()
    if not g.app.globalKillBuffer and not clip_text: return

    undoType = 'yank-pop' if pop else 'yank'
    self.beginCommand(undoType=undoType)
    try:
        if not pop or self.lastYankP and self.lastYankP != current:
            self.reset = 0
        s = self.kbiterator.next()
        if s is None: s = clip_text or ''
        if i != j: w.deleteTextSelection()
        if s != s.lstrip(): # s contains leading whitespace.
            i2,j2 = g.getLine(text,i)
            k = g.skip_ws(text,i2)
            if i2 < i <= k:
                # Replace the line's leading whitespace by s's leading whitespace.
                w.delete(i2,k)
                i = i2
        w.insert(i,s)
        # Fix bug 1099035: Leo yank and kill behaviour not quite the same as emacs.
        # w.setSelectionRange(i,i+len(s),insert=i+len(s))
        w.setInsertPoint(i+len(s))
        self.lastYankP = current.copy()
        c.frame.body.forceFullRecolor()
    finally:
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050930091642.2: *5* yankPop
def yankPop (self,event):

    '''Insert the next entry of the kill ring.'''

    self.yank(event,pop=True)

#@+node:ekr.20050920084036.128: *5* zapToCharacter
def zapToCharacter (self,event):

    '''Kill characters from the insertion point to a given character.'''

    k = self.k ; w = self.editWidget(event)
    if not w: return

    state = k.getState('zap-to-char')
    if state == 0:
        k.setLabelBlue('Zap To Character: ')
        k.setState('zap-to-char',1,handler=self.zapToCharacter)
    else:
        ch = event and event.char or ' '
        k.resetLabel()
        k.clearState()
        s = w.getAllText()
        ins = w.getInsertPoint()
        i = s.find(ch,ins)
        if i == -1: return
        self.beginCommand(undoType='zap-to-char')
        self.addToKillBuffer(s[ins:i])
        g.app.gui.replaceClipboardWith(s[ins:i]) # Support for proper yank.
        w.setAllText(s[:ins] + s[i:])
        w.setInsertPoint(ins)
        self.endCommand(changed=True,setLabel=True)
#@+node:ekr.20050920084036.186: *4* LeoCommandsClass (add docstrings)
class LeoCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20050920084036.187: *5*  ctor (LeoCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.
#@+node:ekr.20050920084036.188: *5* leoCommands.getPublicCommands
def getPublicCommands (self):
    '''(leoCommands) Return a dict of the 'legacy' Leo commands.'''
    c,k = self.c,self.k
    d2 = {}
    << define dictionary d of names and Leo commands >>
    # Create a callback for each item in d.
    for name in sorted(d):
        f = d.get(name)
        d2 [name] = f
        c.inverseCommandsDict [f.__name__] = name
        # g.trace('leoCommands %24s = %s' % (f.__name__,name))
    return d2
#@+node:ekr.20050920084036.189: *6* << define dictionary d of names and Leo commands >>
c = self.c ; f = c.frame

d = {
    'abort-edit-headline':          f.abortEditLabelCommand,
    'about-leo':                    c.about,
    'add-comments':                 c.addComments,     
    'beautify':                     c.beautifyPythonCode,
    'beautify-c':                   c.beautifyCCode,
    'beautify-all':                 c.beautifyAllPythonCode,
    'beautify-tree':                c.beautifyPythonTree,
    'cascade-windows':              f.cascade,
    # 'check-all-python-code':      c.checkAllPythonCode,
    'check-derived-file':           c.atFileCommands.checkDerivedFile,
    'check-leo-file':               c.fileCommands.checkLeoFile,
    'check-outline':                c.checkOutline,
    # 'check-python-code':          c.checkPythonCode,
    'clean-recent-files':           c.cleanRecentFiles,
    'clear-recent-files':           c.clearRecentFiles,
    'clone-node':                   c.clone,
    'clone-node-to-last-node':      c.cloneToLastNode,
    'close-window':                 c.close,
    'contract-all':                 c.contractAllHeadlines,
    'contract-all-other-nodes':     c.contractAllOtherNodes,
    'contract-node':                c.contractNode,
    'contract-or-go-left':          c.contractNodeOrGoToParent,
    'contract-parent':              c.contractParent,
    'convert-all-blanks':           c.convertAllBlanks,
    'convert-all-tabs':             c.convertAllTabs,
    'convert-blanks':               c.convertBlanks,
    'convert-tabs':                 c.convertTabs,
    'copy-node':                    c.copyOutline,
    'copy-text':                    f.copyText,
    'cut-node':                     c.cutOutline,
    'cut-text':                     f.cutText,
    'de-hoist':                     c.dehoist,
    'delete-comments':              c.deleteComments,
    'delete-node':                  c.deleteOutline,
    'demote':                       c.demote,
    'dump-outline':                 c.dumpOutline,
    'edit-headline':                c.editHeadline,
    'end-edit-headline':            f.endEditLabelCommand,
    'equal-sized-panes':            f.equalSizedPanes,
    'execute-script':               c.executeScript,
    'exit-leo':                     g.app.onQuit,
    'expand-all':                   c.expandAllHeadlines,
    'expand-all-subheads':          c.expandAllSubheads,
        # Fixes bug 604037 Status of expandAllSubheads
    'expand-ancestors-only':        c.expandOnlyAncestorsOfNode,
    'expand-and-go-right':          c.expandNodeAndGoToFirstChild,
    'expand-next-level':            c.expandNextLevel,
    'expand-node':                  c.expandNode,
    'expand-or-go-right':           c.expandNodeOrGoToFirstChild,
    'expand-prev-level':            c.expandPrevLevel,
    'expand-to-level-1':            c.expandLevel1,
    'expand-to-level-2':            c.expandLevel2,
    'expand-to-level-3':            c.expandLevel3,
    'expand-to-level-4':            c.expandLevel4,
    'expand-to-level-5':            c.expandLevel5,
    'expand-to-level-6':            c.expandLevel6,
    'expand-to-level-7':            c.expandLevel7,
    'expand-to-level-8':            c.expandLevel8,
    'expand-to-level-9':            c.expandLevel9,
    'export-headlines':             c.exportHeadlines,
    'extract':                      c.extract,
    'extract-names':                c.extractSectionNames,
    # 'extract-python-method':        c.extractPythonMethod,
    # 'extract-section':              c.extractSection,
    'find-next-clone':              c.findNextClone,
    'flatten-outline':              c.flattenOutline,
    'flatten-outline-to-node':      c.flattenOutlineToNode,
    'go-back':                      c.goPrevVisitedNode,
    'go-forward':                   c.goNextVisitedNode,
    'goto-first-node':              c.goToFirstNode,
    'goto-first-sibling':           c.goToFirstSibling,
    'goto-first-visible-node':      c.goToFirstVisibleNode,
    'goto-last-node':               c.goToLastNode,
    'goto-last-sibling':            c.goToLastSibling,
    'goto-last-visible-node':       c.goToLastVisibleNode,
    'goto-next-changed':            c.goToNextDirtyHeadline,
    'goto-next-clone':              c.goToNextClone,
    'goto-next-history-node':       c.goToNextHistory,
    'goto-next-marked':             c.goToNextMarkedHeadline,
    'goto-next-node':               c.selectThreadNext,
    'goto-next-sibling':            c.goToNextSibling,
    'goto-next-visible':            c.selectVisNext,
    'goto-parent':                  c.goToParent,
    'goto-prev-history-node':       c.goToPrevHistory,
    'goto-prev-node':               c.selectThreadBack,
    'goto-prev-sibling':            c.goToPrevSibling,
    'goto-prev-visible':            c.selectVisBack,
    'hide-invisibles':              c.hideInvisibles,
    'hoist':                        c.hoist,
    'import-file':                  c.importAnyFile,
    # 'import-at-file':               c.importAtFile,
    # 'import-at-root':               c.importAtRoot,
    # 'import-cweb-files':            c.importCWEBFiles,
    # 'import-derived-file':          c.importDerivedFile,
    # 'import-flattened-outline':     c.importFlattenedOutline,
    # 'import-noweb-files':           c.importNowebFiles,
    'indent-region':                c.indentBody,
    'insert-body-time':             c.insertBodyTime,
    'insert-child':                 c.insertChild,
    'insert-node':                  c.insertHeadline,
    'insert-node-before':            c.insertHeadlineBefore,
    'mark':                         c.markHeadline,
    'mark-changed-items':           c.markChangedHeadlines,
    # 'mark-changed-roots':           c.markChangedRoots,
    # 'mark-clones':                c.markClones,
    'mark-subheads':                c.markSubheads,
    'match-brackets':               c.findMatchingBracket,
    'minimize-all':                 f.minimizeAll,
    'move-outline-down':            c.moveOutlineDown,
    'move-outline-left':            c.moveOutlineLeft,
    'move-outline-right':           c.moveOutlineRight,
    'move-outline-up':              c.moveOutlineUp,
    'new':                          c.new,
    # 'open-compare-window':        c.openCompareWindow,
    'open-cheat-sheet-leo':         c.openCheatSheet,
    'open-leoDocs-leo':             c.leoDocumentation,
    'open-leoPlugins-leo':          c.openLeoPlugins,
    'open-leoSettings-leo':         c.openLeoSettings,
    'open-local-settings':          c.selectAtSettingsNode,
    'open-myLeoSettings-leo':       c.openMyLeoSettings,
    'open-offline-tutorial':        f.leoHelp,
    'open-online-home':             c.leoHome,
    'open-online-toc':              c.openLeoTOC,
    'open-online-tutorials':        c.openLeoTutorials,
    'open-online-videos':           c.openLeoVideos,
    # 'open-online-tutorial':       c.leoTutorial,
    'open-outline':                 c.open,
    'open-python-window':           c.openPythonWindow,
    'open-quickstart-leo':          c.leoQuickStart,
    'open-scripts-leo':             c.openLeoScripts,
    'open-users-guide':             c.openLeoUsersGuide,
    'open-with':                    c.openWith,
    'outline-to-cweb':              c.outlineToCWEB,
    'outline-to-noweb':             c.outlineToNoweb,
    'paste-node':                   c.pasteOutline,
    'paste-retaining-clones':       c.pasteOutlineRetainingClones,
    'paste-text':                   f.pasteText,
    'pretty-print-all-python-code': c.prettyPrintAllPythonCode,
    'pretty-print-python-code':     c.prettyPrintPythonCode,
    'promote':                      c.promote,
    'read-at-auto-nodes':           c.readAtAutoNodes,
    'read-at-file-nodes':           c.readAtFileNodes,
    'read-at-shadow-nodes':         c.readAtShadowNodes,
    'read-file-into-node':          c.readFileIntoNode,
    'read-outline-only':            c.readOutlineOnly,
    'redo':                         c.undoer.redo,
    # 'reformat-body':              c.reformatBody, # 2013/10/02.
    'reformat-paragraph':           c.reformatParagraph,
    'remove-sentinels':             c.removeSentinels,
    'resize-to-screen':             f.resizeToScreen,
    'refresh-from-disk':            c.refreshFromDisk,
    'revert':                       c.revert,
    'save-all':                     c.saveAll,
    'save-file':                    c.save,
    'save-file-as':                 c.saveAs,
    'save-file-as-unzipped':        c.saveAsUnzipped,
    'save-file-as-zipped':          c.saveAsZipped,
    'save-file-to':                 c.saveTo,
    'set-colors':                   c.colorPanel,
    'set-font':                     c.fontPanel,
    'settings':                     c.preferences,
    'show-invisibles':              c.showInvisibles,
    'sort-children':                c.sortChildren,
    'sort-recent-files':            c.sortRecentFiles,
    'sort-siblings':                c.sortSiblings,
    'tangle':                       c.tangle,
    'tangle-all':                   c.tangleAll,
    'tangle-marked':                c.tangleMarked,
    'toggle-active-pane':           f.toggleActivePane,
    'toggle-angle-brackets':        c.toggleAngleBrackets,
    'toggle-invisibles':            c.toggleShowInvisibles,
    'toggle-sparse-move':           c.toggleSparseMove,
    'toggle-split-direction':       f.toggleSplitDirection,
    'undo':                         c.undoer.undo,
    'unindent-region':              c.dedentBody,
    'unmark-all':                   c.unmarkAll,
    'untangle':                     c.untangle,
    'untangle-all':                 c.untangleAll,
    'untangle-marked':              c.untangleMarked,
    'weave':                        c.weave,
    'write-at-auto-nodes':          c.atFileCommands.writeAtAutoNodes,
    'write-at-file-nodes':          c.fileCommands.writeAtFileNodes,
    'write-at-shadow-nodes':        c.fileCommands.writeAtShadowNodes,
    'write-dirty-at-auto-nodes':    c.atFileCommands.writeDirtyAtAutoNodes,
    'write-dirty-at-file-nodes':    c.fileCommands.writeDirtyAtFileNodes,
    'write-dirty-at-shadow-nodes':  c.fileCommands.writeDirtyAtShadowNodes,
    'write-file-from-node':         c.writeFileFromNode,
    'write-missing-at-file-nodes':  c.fileCommands.writeMissingAtFileNodes,
    'write-outline-only':           c.fileCommands.writeOutlineOnly,
}
#@+node:ekr.20050920084036.221: *4* RecTangleCommandsClass
class RecTangleCommandsClass (BaseEditCommandsClass):

    @others
#@+node:ekr.20050920084036.222: *5*  Birth (RecTangleCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.theKillRectangle = [] # Do not re-init this!
    self.stringRect = None

def finishCreate(self):

    BaseEditCommandsClass.finishCreate(self)

    self.commandsDict = {
        'c': ('clear-rectangle',    self.clearRectangle),
        'd': ('delete-rectangle',   self.deleteRectangle),
        'k': ('kill-rectangle',     self.killRectangle),
        'o': ('open-rectangle',     self.openRectangle),
        'r': ('copy-rectangle-to-register',
            self.c.registerCommands.copyRectangleToRegister),
        't': ('string-rectangle',   self.stringRectangle),
        'y': ('yank-rectangle',     self.yankRectangle),
    }
#@+node:ekr.20051004112630: *5* check
def check (self,event,warning='No rectangle selected'):

    '''Return True if there is a selection.
    Otherwise, return False and issue a warning.'''

    return self._chckSel(event,warning)
#@+node:ekr.20050920084036.223: *5* getPublicCommands
def getPublicCommands (self):

    return {
        'rectangle-clear':  self.clearRectangle,
        'rectangle-close':  self.closeRectangle,
        'rectangle-delete': self.deleteRectangle,
        'rectangle-kill':   self.killRectangle,
        'rectangle-open':   self.openRectangle,
        'rectangle-string': self.stringRectangle,
        'rectangle-yank':   self.yankRectangle,
    }
#@+node:ekr.20051215103053: *5* beginCommand & beginCommandWithEvent (rectangle)
def beginCommand (self,undoType='Typing'):

    w = BaseEditCommandsClass.beginCommand(self,undoType)
    r1,r2,r3,r4 = self.getRectanglePoints(w)
    return w,r1,r2,r3,r4


def beginCommandWithEvent (self,event,undoType='Typing'):

    '''Do the common processing at the start of each command.'''

    w = BaseEditCommandsClass.beginCommandWithEvent(self,event,undoType)
    r1,r2,r3,r4 = self.getRectanglePoints(w)
    return w,r1,r2,r3,r4
#@+node:ekr.20050920084036.224: *5* Entries (RecTangleCommandsClass)
#@+node:ekr.20050920084036.225: *6* clearRectangle
def clearRectangle (self,event):

    '''Clear the rectangle defined by the start and end of selected text.'''

    w = self.editWidget(event)
    if not w or not self.check(event): return

    w,r1,r2,r3,r4 = self.beginCommand('clear-rectangle')

    # Change the text.
    fill = ' ' *(r4-r2)
    for r in range(r1,r3+1):
        w.delete('%s.%s' % (r,r2),'%s.%s' % (r,r4))
        w.insert('%s.%s' % (r,r2),fill)

    w.setSelectionRange('%s.%s'%(r1,r2),'%s.%s'%(r3,r2+len(fill)))

    self.endCommand()
#@+node:ekr.20050920084036.226: *6* closeRectangle
def closeRectangle (self,event):

    '''Delete the rectangle if it contains nothing but whitespace..'''

    w = self.editWidget(event)
    if not w or not self.check(event): return

    w,r1,r2,r3,r4 = self.beginCommand('close-rectangle')

    # Return if any part of the selection contains something other than whitespace.
    for r in range(r1,r3+1):
        s = w.get('%s.%s' % (r,r2),'%s.%s' % (r,r4))
        if s.strip(): return

    # Change the text.
    for r in range(r1,r3+1):
        w.delete('%s.%s' % (r,r2),'%s.%s' % (r,r4))

    i = '%s.%s' % (r1,r2)
    j = '%s.%s' % (r3,r2)
    w.setSelectionRange(i,j,insert=j)

    self.endCommand()
#@+node:ekr.20050920084036.227: *6* deleteRectangle
def deleteRectangle (self,event):

    '''Delete the rectangle defined by the start and end of selected text.'''

    w = self.editWidget(event)
    if not w or not self.check(event): return

    w,r1,r2,r3,r4 = self.beginCommand('delete-rectangle')

    for r in range(r1,r3+1):
        w.delete('%s.%s' % (r,r2),'%s.%s' % (r,r4))

    i = '%s.%s' % (r1,r2)
    j = '%s.%s' % (r3,r2)
    w.setSelectionRange(i,j,insert=j)

    self.endCommand()
#@+node:ekr.20050920084036.228: *6* killRectangle
def killRectangle (self,event):

    '''Kill the rectangle defined by the start and end of selected text.'''

    w = self.editWidget(event)
    if not w or not self.check(event): return

    w,r1,r2,r3,r4 = self.beginCommand('kill-rectangle')

    self.theKillRectangle = []

    for r in range(r1,r3+1):
        s = w.get('%s.%s' % (r,r2),'%s.%s' % (r,r4))
        self.theKillRectangle.append(s)
        w.delete('%s.%s' % (r,r2),'%s.%s' % (r,r4))

    # g.trace('killRect',repr(self.theKillRectangle))

    if self.theKillRectangle:
        ins = '%s.%s' % (r,r2)
        w.setSelectionRange(ins,ins,insert=ins)

    self.endCommand()
#@+node:ekr.20050920084036.230: *6* openRectangle
def openRectangle (self,event):

    '''Insert blanks in the rectangle defined by the start and end of selected text.
    This pushes the previous contents of the rectangle rightward.'''

    w = self.editWidget(event)
    if not w or not self.check(event): return

    w,r1,r2,r3,r4 = self.beginCommand('open-rectangle')

    fill = ' ' * (r4-r2)
    for r in range(r1,r3+1):
        w.insert('%s.%s' % (r,r2),fill)

    i = '%s.%s' % (r1,r2)
    j = '%s.%s' % (r3,r2+len(fill))
    w.setSelectionRange(i,j,insert=j)

    self.endCommand()
#@+node:ekr.20050920084036.232: *6* stringRectangle
def stringRectangle (self,event):

    '''Prompt for a string, then replace the contents of a rectangle
    with a string on each line.'''

    c = self.c ; k = self.k ; state = k.getState('string-rect')
    if g.app.unitTesting:
        state = 1 ; k.arg = 's...s' # This string is known to the unit test.
        w = self.editWidget(event)
        self.stringRect = self.getRectanglePoints(w)
    if state == 0:
        w = self.editWidget(event) # sets self.w
        if not w or not self.check(event): return
        self.stringRect = self.getRectanglePoints(w)
        k.setLabelBlue('String rectangle: ')
        k.getArg(event,'string-rect',1,self.stringRectangle)
    else:
        k.clearState()
        k.resetLabel()
        c.bodyWantsFocus()
        w = self.w
        self.beginCommand('string-rectangle')
        # pylint: disable=unpacking-non-sequence
        r1,r2,r3,r4 = self.stringRect
        s = w.getAllText()
        for r in range(r1,r3+1):
            i = g.convertRowColToPythonIndex(s,r-1,r2)
            j = g.convertRowColToPythonIndex(s,r-1,r4)
            s = s[:i] + k.arg + s[j:]
        w.setAllText(s)
        i = g.convertRowColToPythonIndex(s,r1-1,r2)
        j = g.convertRowColToPythonIndex(s,r3-1,r2+len(k.arg))
        w.setSelectionRange(i,j)
        self.endCommand()
        # 2010/1/1: Fix bug 480422:
        # string-rectangle kills syntax highlighting.
        c.frame.body.recolor(c.p,incremental=False)

#@+node:ekr.20050920084036.229: *6* yankRectangle
def yankRectangle (self,event,killRect=None):

    '''Yank into the rectangle defined by the start and end of selected text.'''

    # c = self.c
    k = self.k
    w = self.editWidget(event)
    if not w: return

    killRect = killRect or self.theKillRectangle
    if g.app.unitTesting:
        # This value is used by the unit test.
        killRect = ['Y1Y','Y2Y','Y3Y','Y4Y']
    elif not killRect:
        k.setLabelGrey('No kill rect') ; return

    w,r1,r2,r3,r4 = self.beginCommand('yank-rectangle')

    n = 0
    for r in range(r1,r3+1):
        # g.trace(n,r,killRect[n])
        if n >= len(killRect): break
        w.delete('%s.%s' % (r,r2), '%s.%s' % (r,r4))
        w.insert('%s.%s' % (r,r2), killRect[n])
        n += 1

    i = '%s.%s' % (r1,r2)
    j = '%s.%s' % (r3,r2+len(killRect[n-1]))
    w.setSelectionRange(i,j,insert=j)
    self.endCommand()
#@+node:ekr.20050920084036.234: *4* RegisterCommandsClass
class RegisterCommandsClass (BaseEditCommandsClass):

    '''A class to represent registers a-z and the corresponding Emacs commands.'''

    @others
#@+node:ekr.20051004095209: *5* Birth
#@+node:ekr.20050920084036.235: *6*  Birth (RegisterCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.
    self.methodDict, self.helpDict = self.addRegisterItems()

    # Init these here to keep pylint happy.
    self.method = None 
    self.registerMode = 0 # Must be an int.
    self.registers = g.app.globalRegisters

def finishCreate (self):

    BaseEditCommandsClass.finishCreate(self)
        # finish the base class.

def init (self):
    self.method = None 
    self.registerMode = 0 # Must be an int.
    self.registers = {}

#@+node:ekr.20050920084036.247: *6*  getPublicCommands
def getPublicCommands (self):

    return {
        'register-append-to':           self.appendToRegister,
        'register-copy-rectangle-to':   self.copyRectangleToRegister,
        'register-copy-to':             self.copyToRegister,
        'register-increment':           self.incrementRegister,
        'register-insert':              self.insertRegister,
        'register-jump-to':             self.jumpToRegister,
        # 'register-number-to':         self.numberToRegister,
        'register-point-to':            self.pointToRegister,
        'register-prepend-to':          self.prependToRegister,
        'register-view':                self.viewRegister,
    }
#@+node:ekr.20050920084036.252: *6* addRegisterItems
def addRegisterItems( self ):

    methodDict = {
        '+':        self.incrementRegister,
        ' ':        self.pointToRegister,
        'a':        self.appendToRegister,
        'i':        self.insertRegister,
        'j':        self.jumpToRegister,
        # 'n':        self.numberToRegister,
        'p':        self.prependToRegister,
        'r':        self.copyRectangleToRegister,
        's':        self.copyToRegister,
        'v' :       self.viewRegister,
    }    

    helpDict = {
        's':    'copy to register',
        'i':    'insert from register',
        '+':    'increment register',
        'n':    'number to register',
        'p':    'prepend to register',
        'a':    'append to register',
        ' ':    'point to register',
        'j':    'jump to register',
        'r':    'rectangle to register',
        'v':    'view register',
    }

    return methodDict, helpDict
#@+node:ekr.20051004123217: *5* checkBodySelection
def checkBodySelection (self,warning='No text selected'):

    return self._chckSel(event=None,warning=warning)
#@+node:ekr.20050920084036.236: *5* Entries... (register commands)
#@+node:ekr.20050920084036.238: *6* appendToRegister
def appendToRegister (self,event):

    '''Prompt for a register name and append the selected text to the register's contents.'''

    c = self.c ; k = self.k
    tag = 'append-to-register' ; state = k.getState(tag)

    char = event and event.char or ''

    if state == 0:
        k.commandName = tag
        k.setLabelBlue('Append to Register: ')
        k.setState(tag,1,self.appendToRegister)
    else:
        k.clearState()
        if self.checkBodySelection():
            if char.isalpha():
                w = c.frame.body.wrapper
                c.bodyWantsFocus()
                key = char.lower()
                val = self.registers.get(key,'')
                val = val + w.getSelectedText()
                self.registers[key] = val
                k.setLabelGrey('Register %s = %s' % (key,repr(val)))
            else:
                k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.237: *6* prependToRegister
def prependToRegister (self,event):

    '''Prompt for a register name and prepend the selected text to the register's contents.'''

    c = self.c ; k = self.k
    tag = 'prepend-to-register' ; state = k.getState(tag)

    char = event and event.char or ''

    if state == 0:
        k.commandName = tag
        k.setLabelBlue('Prepend to Register: ')
        k.setState(tag,1,self.prependToRegister)
    else:
        k.clearState()
        if self.checkBodySelection():
            if char.isalpha():
                w = c.frame.body.wrapper
                c.bodyWantsFocus()
                key = char.lower()
                val = self.registers.get(key,'')
                val = w.getSelectedText() + val
                self.registers[key] = val
                k.setLabelGrey('Register %s = %s' % (key,repr(val)))
            else:
                k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.239: *6* copyRectangleToRegister
def copyRectangleToRegister (self,event):

    '''Prompt for a register name and append the rectangle defined by selected
    text to the register's contents.'''

    c = self.c ; k = self.k ; state = k.getState('copy-rect-to-reg')

    char = event and event.char or ''

    if state == 0:
        w = self.editWidget(event) # sets self.w
        if not w: return
        k.commandName = 'copy-rectangle-to-register'
        k.setLabelBlue('Copy Rectangle To Register: ')
        k.setState('copy-rect-to-reg',1,self.copyRectangleToRegister)
    elif self.checkBodySelection('No rectangle selected'):
        k.clearState()
        if char.isalpha():
            key = char.lower()
            w = self.w
            c.widgetWantsFocusNow(w)
            r1, r2, r3, r4 = self.getRectanglePoints(w)
            rect = []
            while r1 <= r3:
                txt = w.get('%s.%s' % (r1,r2),'%s.%s' % (r1,r4))
                rect.append(txt)
                r1 = r1 + 1
            self.registers [key] = rect
            k.setLabelGrey('Register %s = %s' % (key,repr(rect)))
        else:
            k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.240: *6* copyToRegister
def copyToRegister (self,event):

    '''Prompt for a register name and append the selected text to the register's contents.'''

    c = self.c ; k = self.k
    tag = 'copy-to-register' ; state = k.getState(tag)

    char = event and event.char or ''

    if state == 0:
        k.commandName = tag
        k.setLabelBlue('Copy to Register: ')
        k.setState(tag,1,self.copyToRegister)
    else:
        k.clearState()
        if self.checkBodySelection():
            if char.isalpha():
                key = char.lower()
                w = c.frame.body.wrapper
                c.bodyWantsFocus()
                val = w.getSelectedText()
                self.registers[key] = val
                k.setLabelGrey('Register %s = %s' % (key,repr(val)))
            else:
                k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.241: *6* incrementRegister
def incrementRegister (self,event):

    '''Prompt for a register name and increment its value if it has a numeric value.'''

    c = self.c ; k = self.k ; state = k.getState('increment-reg')

    char = event and event.char or ''

    if state == 0:
        k.setLabelBlue('Increment register: ')
        k.setState('increment-reg',1,self.incrementRegister)
    else:
        k.clearState()
        if self._checkIfRectangle(event):
            pass # Error message is in the label.
        elif char.isalpha():
            key = char.lower()
            val = self.registers.get(key,0)
            try:
                val = str(int(val)+1)
                self.registers[key] = val
                k.setLabelGrey('Register %s = %s' % (key,repr(val)))
            except ValueError:
                k.setLabelGrey("Can't increment register %s = %s" % (key,val))
        else:
            k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.242: *6* insertRegister
def insertRegister (self,event):

    '''Prompt for a register name and and insert the value of another register into its contents.'''

    c = self.c ; k = self.k ; state = k.getState('insert-reg')

    char = event and event.char or ''

    if state == 0:
        k.commandName = 'insert-register'
        k.setLabelBlue('Insert register: ')
        k.setState('insert-reg',1,self.insertRegister)
    else:
        k.clearState()
        if char.isalpha():
            w = c.frame.body.wrapper
            c.bodyWantsFocus()
            key = char.lower()
            val = self.registers.get(key)
            if val:
                if type(val)==type([]):
                    c.recTangleCommands.yankRectangle(val)
                else:
                    i = w.getInsertPoint()
                    w.insert(i,val)
                k.setLabelGrey('Inserted register %s' % key)
            else:
                k.setLabelGrey('Register %s is empty' % key)
        else:
            k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.243: *6* jumpToRegister
def jumpToRegister (self,event):

    '''Prompt for a register name and set the insert point to the value in its register.'''

    c = self.c ; k = self.k ; state = k.getState('jump-to-reg')

    char = event and event.char or ''

    if state == 0:
        k.setLabelBlue('Jump to register: ')
        k.setState('jump-to-reg',1,self.jumpToRegister)
    else:
        k.clearState()
        if char.isalpha():
            if self._checkIfRectangle(event): return
            key = char.lower()
            val = self.registers.get(key)
            w = c.frame.body.wrapper
            c.bodyWantsFocus()
            if val:
                try:
                    w.setInsertPoint(val)
                    k.setLabelGrey('At %s' % repr(val))
                except Exception:
                    k.setLabelGrey('Register %s is not a valid location' % key)
            else:
                k.setLabelGrey('Register %s is empty' % key)
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.244: *6* numberToRegister (not used)
@
C-u number C-x r n reg
    Store number into register reg (number-to-register).
C-u number C-x r + reg
    Increment the number in register reg by number (increment-register).
C-x r g reg
    Insert the number from register reg into the buffer.
@c

def numberToRegister (self,event):

    c,k = self.c,self.k
    state = k.getState('number-to-reg')

    char = event and event.char or ''

    if state == 0:
        k.commandName = 'number-to-register'
        k.setLabelBlue('Number to register: ')
        k.setState('number-to-reg',1,self.numberToRegister)
    else:
        k.clearState()
        if char.isalpha():
            # self.registers[char.lower()] = str(0)
            k.setLabelGrey('number-to-register not ready yet.')
        else:
            k.setLabelGrey('Register must be a letter')
#@+node:ekr.20050920084036.245: *6* pointToRegister
def pointToRegister (self,event):

    '''Prompt for a register name and put a value indicating the insert point in the register.'''

    c = self.c ; k = self.k ; state = k.getState('point-to-reg')

    char = event and event.char or ''

    if state == 0:
        k.commandName = 'point-to-register'
        k.setLabelBlue('Point to register: ')
        k.setState('point-to-reg',1,self.pointToRegister)
    else:
        k.clearState()
        if char.isalpha():
            w = c.frame.body.wrapper
            c.bodyWantsFocus()
            key = char.lower()
            val = w.getInsertPoint()
            self.registers[key] = val
            k.setLabelGrey('Register %s = %s' % (key,repr(val)))
        else:
            k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20050920084036.246: *6* viewRegister
def viewRegister (self,event):

    '''Prompt for a register name and print its contents.'''

    c = self.c ; k = self.k ; state = k.getState('view-reg')

    char = event and event.char or ''

    if state == 0:
        k.commandName = 'view-register'
        k.setLabelBlue('View register: ')
        k.setState('view-reg',1,self.viewRegister)
    else:
        k.clearState()
        if char.isalpha():
            key = char.lower()
            val = self.registers.get(key)
            k.setLabelGrey('Register %s = %s' % (key,repr(val)))
        else:
            k.setLabelGrey('Register must be a letter')
    c.bodyWantsFocus()
#@+node:ekr.20051023094009: *4* Search classes (leoEditCommands)
#@+node:ekr.20050920084036.257: *5* class SearchCommandsClass
class SearchCommandsClass (BaseEditCommandsClass):

    '''Delegates all searches to LeoFind.py.'''
    
    if 0: # Not needed.
        def __init__ (self,c):
            BaseEditCommandsClass.__init__(self,c)

    @others
#@+node:ekr.20050920084036.259: *6* getPublicCommands (SearchCommandsClass)
def getPublicCommands (self):
    
    find = self.c.findCommands
    return {
    'clone-find-all':                 find.minibufferCloneFindAll,
    'clone-find-all-flattened':       find.minibufferCloneFindAllFlattened,
    'clone-find-parents':             self.c.cloneFindParents,
    'find-all':                       find.minibufferFindAll,
    'find-clone-all':                 find.minibufferCloneFindAll,
    'find-clone-all-flattened':       find.minibufferCloneFindAllFlattened,
    'find-next':                      find.findNextCommand,
    'find-prev':                      find.findPrevCommand,
    'find-tab-hide':                  find.hideFindTab,
    'find-tab-open':                  find.openFindTab,
    'focus-to-find':                  find.focusToFind,
    'isearch-forward':                find.isearchForward,
    'isearch-backward':               find.isearchBackward,
    'isearch-forward-regexp':         find.isearchForwardRegexp,
    'isearch-backward-regexp':        find.isearchBackwardRegexp,
    'isearch-with-present-options':   find.isearchWithPresentOptions,
    'replace':                        find.change,
    'replace-all':                    find.minibufferReplaceAll,
    # 'replace-string':               find.setReplaceString,
    'replace-then-find':              find.changeThenFindCommand,
    're-search-forward':              find.reSearchForward,
    're-search-backward':             find.reSearchBackward,
    'search-forward':                 find.searchForward,
    'search-backward':                find.searchBackward,
    'search-with-present-options':    find.searchWithPresentOptions,
    'set-search-string':              find.searchWithPresentOptions,
    'set-replace-string':             find.setReplaceString,
    'set-find-everywhere':            find.setFindScopeEveryWhere,
    'set-find-node-only':             find.setFindScopeNodeOnly,
    'set-find-suboutline-only':       find.setFindScopeSuboutlineOnly,
    'show-find-options':              find.showFindOptions,
    'start-search':                   find.startSearch, # 4.11.1.
    'toggle-find-collapses-nodes':    find.toggleFindCollapesNodes,
    'toggle-find-ignore-case-option': find.toggleIgnoreCaseOption,
    'toggle-find-in-body-option':     find.toggleSearchBodyOption,
    'toggle-find-in-headline-option': find.toggleSearchHeadlineOption,
    'toggle-find-mark-changes-option':find.toggleMarkChangesOption,
    'toggle-find-mark-finds-option':  find.toggleMarkFindsOption,
    'toggle-find-regex-option':       find.toggleRegexOption,
    'toggle-find-word-option':        find.toggleWholeWordOption,
    'toggle-find-wrap-around-option': find.toggleWrapSearchOption,
    'word-search-forward':            find.wordSearchForward,
    'word-search-backward':           find.wordSearchBackward,
    }
#@+node:ekr.20051025071455: *4* Spell classes (leoEditCommands)
@others
#@+node:ekr.20051025071455.1: *5* class SpellCommandsClass
class SpellCommandsClass (BaseEditCommandsClass):

    '''Commands to support the Spell Tab.'''

    @others
#@+node:ekr.20051025080056: *6* ctor (SpellCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.handler = None

    # All the work happens when we first open the frame.
#@+node:ekr.20051025080420: *6* getPublicCommands (SearchCommandsClass)
def getPublicCommands (self):

    return {
        'focus-to-spell':           self.focusToSpell,
        'open-spell-tab':           self.openSpellTab,
        'spell-find':               self.find,
        'spell-change':             self.change,
        'spell-change-then-find':   self.changeThenFind,
        'spell-ignore':             self.ignore,
        'hide-spell-tab':           self.hide,
        
        # these are for spell as you type, not the spell tab
        'spell-as-you-type-toggle': self.as_you_type_toggle,
        'spell-as-you-type-wrap':   self.as_you_type_wrap,
        'spell-as-you-type-next':   self.as_you_type_next,
        'spell-as-you-type-undo':   self.as_you_type_undo,        
    }
#@+node:ekr.20051025080633: *6* openSpellTab
def openSpellTab (self,event=None):

    '''Open the Spell Checker tab in the log pane.'''

    c = self.c ; log = c.frame.log ; tabName = 'Spell'

    if log.frameDict.get(tabName):
        log.selectTab(tabName)
    else:
        log.selectTab(tabName)
        self.handler = SpellTabHandler(c,tabName)

    # Bug fix: 2013/05/22.
    if not self.handler.loaded:
        log.deleteTab(tabName,force=True)

    # spell as you type stuff
    self.suggestions = []
    self.suggestions_idx = None
    self.word = None
    self.spell_as_you_type = False
    self.wrap_as_you_type = False
#@+node:ekr.20051025080420.1: *6* commands...(SpellCommandsClass)
#@+node:ekr.20141113094129.8: *7* find
def find (self,event=None):
    '''
    Simulate pressing the 'Find' button in the Spell tab.
    
    Just open the Spell tab if it has never been opened.
    For minibuffer commands, we must also force the Spell tab to be visible.
    '''
    # self.handler is a SpellTabHandler object (inited by openSpellTab)
    if self.handler:
        self.openSpellTab()
        self.handler.find()
    else:
        self.openSpellTab()

#@+node:ekr.20141113094129.9: *7* change
def change(self,event=None):
    '''Simulate pressing the 'Change' button in the Spell tab.'''
    if self.handler:
        self.openSpellTab()
        self.handler.change()
    else:
        self.openSpellTab()

#@+node:ekr.20141113094129.10: *7* changeThenFind
def changeThenFind (self,event=None):
    '''Simulate pressing the 'Change, Find' button in the Spell tab.'''
    if self.handler:
        self.openSpellTab()
        # A workaround for a pylint warning:
        # self.handler.changeThenFind()
        f = getattr(self.handler,'changeThenFind')
        f()
    else:
        self.openSpellTab()

#@+node:ekr.20141113094129.11: *7* hide
def hide (self,event=None):
    '''Hide the Spell tab.'''
    if self.handler:
        self.c.frame.log.selectTab('Log')
        self.c.bodyWantsFocus()

#@+node:ekr.20141113094129.12: *7* ignore
def ignore (self,event=None):
    '''Simulate pressing the 'Ignore' button in the Spell tab.'''
    if self.handler:
        self.openSpellTab()
        self.handler.ignore()
    else:
        self.openSpellTab()
#@+node:ekr.20141113094129.5: *7* focusToSpell
def focusToSpell(self,event=None):
    '''Put focus in the spell tab.'''
    self.openSpellTab()
        # Makes Spell tab visible.
        
    # This is not a great idea. There is no indication of focus.
        # if self.handler and self.handler.tab:
            # self.handler.tab.setFocus()
#@+node:tbrown.20140117115926.30765: *6* as_you_type_* commands
#@+node:tbrown.20140117115926.30766: *7* as_you_type_toggle
def as_you_type_toggle(self, event):
    """as_you_type_toggle - toggle spell as you type

    :Parameters:
    - `event`: event triggering toggle, not useful
    """

    c = self.c
    if self.spell_as_you_type:
        self.spell_as_you_type = False
        if not self.wrap_as_you_type:
            g.unregisterHandler('bodykey2', self.as_you_type_onkey)
        g.es("Spell as you type disabled")
        return
    
    self.spell_as_you_type = True
    if not self.wrap_as_you_type:
        g.registerHandler('bodykey2', self.as_you_type_onkey)
    g.es("Spell as you type enabled")
#@+node:tbrown.20140805135321.39151: *7* as_you_type_wrap
def as_you_type_wrap(self, event):
    """as_you_type_wrap - toggle wrap as you type

    :Parameters:
    - `event`: event triggering toggle, not useful
    """

    c = self.c
    if self.wrap_as_you_type:
        self.wrap_as_you_type = False
        if not self.spell_as_you_type:
            g.unregisterHandler('bodykey2', self.as_you_type_onkey)
        g.es("Wrap as you type disabled")
        return
    
    self.wrap_as_you_type = True
    if not self.spell_as_you_type:
        g.registerHandler('bodykey2', self.as_you_type_onkey)
    g.es("Wrap as you type enabled")
#@+node:tbrown.20140117115926.30768: *7* as_you_type_next
def as_you_type_next(self, event):
    """as_you_type_next - cycle word behind cursor to next suggestion

    :Parameters:
    - `event`: triggering key event
    """

    if not self.suggestions:
        g.es('[no suggestions]')
        return
    word = self.suggestions[self.suggestion_idx]
    self.suggestion_idx = (self.suggestion_idx + 1) % len(self.suggestions)
    self.as_you_type_replace(word)
    
#@+node:tbrown.20140117115926.30770: *7* as_you_type_undo
def as_you_type_undo(self, event):
    """as_you_type_undo - replace word behind cursor with word
    user typed before it started cycling suggestions

    :Parameters:
    - `event`: triggering event
    """

    if not self.word:
        g.es('[no previous word]')
        return
    self.as_you_type_replace(self.word)
#@+node:tbrown.20140117115926.30771: *7* as_you_type_onkey
def as_you_type_onkey(self, tag, kwargs):
    """as_you_type_onkey - handle a keystroke in the body when
    spell as you type is active

    :Parameters:
    - `tag`: hook tag
    - `kwargs`: hook arguments
    """
    if kwargs['c'] != self.c:
        return
    if kwargs['ch'] not in '\'",.:) \n\t':
        return
    c = self.c
    spell_ok = True
    if self.spell_as_you_type:  # might just be for wrapping
        w = c.frame.body.wrapper
        txt = w.getAllText()
        i = w.getInsertPoint()
        word = txt[:i].rsplit(None, 1)[-1]
        word = ''.join(i if i.isalpha() else ' ' for i in word).split()
        if word:
            word = word[-1]
            ec = c.spellCommands.handler.spellController
            suggests = ec.processWord(word)
            if suggests:
                spell_ok = False
                g.es(' '.join(suggests[:5]) +
                     ('...' if len(suggests) > 5 else ''),
                     color='red')
            elif suggests is not None:
                spell_ok = False
                g.es('[no suggestions]')
            self.suggestions = suggests
            self.suggestion_idx = 0
            self.word = word
    if spell_ok and self.wrap_as_you_type and kwargs['ch'] == '\n':
        g.es('filling')
        c.k.simulateCommand('fill-paragraph')
#@+node:tbrown.20140117133522.32004: *7* as_you_type_replace
def as_you_type_replace(self, word):
    """as_you_type_replace - replace the word behind the cursor
    with `word`

    :Parameters:
    - `word`: word to use as replacement
    """
    c = self.c
    w = c.frame.body.wrapper
    txt = w.getAllText()
    j = i = w.getInsertPoint()
    i -= 1
    while i and not txt[i].isalpha():
        i -= 1
    xtra = j - i
    j = i+1
    while i and txt[i].isalpha():
        i -= 1
    if i or (txt and not txt[0].isalpha()):
        i += 1
    txt = txt[:i]+word+txt[j:]
    w.setAllText(txt)
    c.p.b = txt
    w.setInsertPoint(i+len(word)+xtra-1)
    c.bodyWantsFocusNow()
#@+node:ekr.20051025071455.18: *5* class SpellTabHandler
class SpellTabHandler:

    """A class to create and manage Leo's Spell Check dialog."""

    @others
#@+node:ekr.20051025071455.19: *6* Birth & death
#@+node:ekr.20051025071455.20: *7* SpellTabHandler.__init__
def __init__(self,c,tabName):
    """Ctor for the Leo Spelling dialog."""
    self.c = c
    self.body = c.frame.body
    self.currentWord = None
    self.outerScrolledFrame = None
    self.workCtrl = g.app.gui.plainTextWidget(c.frame.top)
        # A text widget for scanning.
        # Must have a parent frame even though it is not packed.
    if enchant:
        self.spellController = EnchantClass(c)
        self.tab = g.app.gui.createSpellTab(c,self,tabName)
        self.loaded = True
    else:
        self.spellController = None
        self.tab = None
        self.loaded = False
#@+node:ekr.20051025071455.36: *6* Commands
#@+node:ekr.20051025071455.37: *7* add (spellTab)
def add(self,event=None):
    """Add the selected suggestion to the dictionary."""
    if self.loaded:
        w = self.currentWord
        if w:
            self.spellController.add(w)
            self.tab.onFindButton()
#@+node:ekr.20051025071455.38: *7* change (spellTab)
def change(self,event=None):
    """Make the selected change to the text"""
    if not self.loaded:
        return
    c = self.c
    w = c.frame.body.wrapper
    selection = self.tab.getSuggestion()
    if selection:
        # Use getattr to keep pylint happy.
        if hasattr(self.tab,'change_i') and getattr(self.tab,'change_i') is not None:
            start = getattr(self.tab,'change_i')
            end   = getattr(self.tab,'change_j')
            oldSel = start,end
            # g.trace('using',start,end)
        else:
            start,end = oldSel = w.getSelectionRange()
        if start is not None:
            if start > end: start,end = end,start
            w.delete(start,end)
            w.insert(start,selection)
            w.setSelectionRange(start,start+len(selection))
            c.frame.body.onBodyChanged("Change",oldSel=oldSel)
            c.invalidateFocus()
            c.bodyWantsFocus()
            return True
    # The focus must never leave the body pane.
    c.invalidateFocus()
    c.bodyWantsFocus()
    return False
#@+node:ekr.20051025071455.40: *7* find & helpers
def find (self,event=None):
    """Find the next unknown word."""
    if not self.loaded:
        return
    c = self.c
    w = c.frame.body.wrapper
    # Reload the work pane from the present node.
    s = w.getAllText().rstrip()
    self.workCtrl.delete(0,"end")
    self.workCtrl.insert("end",s)
    # Reset the insertion point of the work widget.
    ins = w.getInsertPoint()
    self.workCtrl.setInsertPoint(ins)
    alts, word = self.findNextMisspelledWord()
    self.currentWord = word # Need to remember this for 'add' and 'ignore'
    if alts:
        # Save the selection range.
        ins = w.getInsertPoint()
        i,j = w.getSelectionRange()
        self.tab.fillbox(alts,word)
        c.invalidateFocus()
        c.bodyWantsFocus()
        # Restore the selection range.
        w.setSelectionRange(i,j,insert=ins)
        w.see(ins)
    else:
        g.es("no more misspellings")
        self.tab.fillbox([])
        c.invalidateFocus()
        c.bodyWantsFocus()
#@+node:ekr.20051025071455.45: *8* findNextMisspelledWord
def findNextMisspelledWord(self):
    """Find the next unknown word."""
    trace = False and not g.unitTesting
    c = self.c ; p = c.p
    w = c.frame.body.wrapper
    sc = self.spellController
    alts = None ; word = None
    try:
        while 1:
            i,j,p,word = self.findNextWord(p)
            # g.trace(i,j,p and p.h or '<no p>')
            if not p or not word:
                alts = None
                break
            alts = sc.processWord(word)
            if trace: g.trace('alts',alts and len(alts) or 0,i,j,word,p and p.h or 'None')
            if alts:
                redraw = not p.isVisible(c)
                # New in Leo 4.4.8: show only the 'sparse' tree when redrawing.
                if c.sparse_spell and not c.p.isAncestorOf(p):
                    for p2 in c.p.self_and_parents():
                        p2.contract()
                        redraw = True
                for p2 in p.parents():
                    if not p2.isExpanded():
                        p2.expand()
                        redraw = True
                if redraw:
                    c.redraw(p)
                else:
                    c.selectPosition(p)
                w.setSelectionRange(i,j,insert=j)
                break
    except Exception:
        g.es_exception()
    return alts, word
#@+node:ekr.20051025071455.47: *8* findNextWord (spellTab)
def findNextWord(self,p):
    """Scan for the next word, leaving the result in the work widget"""
    trace = False and not g.unitTesting
    c = self.c ; p = p.copy()
    while 1:
        s = self.workCtrl.getAllText()
        i = self.workCtrl.getInsertPoint()
        while i < len(s) and not g.isWordChar1(s[i]):
            i += 1
        # g.trace('p',p and p.h,'i',i,'len(s)',len(s))
        if i < len(s):
            # A non-empty word has been found.
            j = i
            while j < len(s) and (g.isWordChar(s[j]) or s[j] == "'"):
                j += 1
            word = s[i:j]
            word = word.strip("'")
            # This trace verifies that all words have been checked.
            # g.trace(repr(word))
            for w in (self.workCtrl,c.frame.body.wrapper):
                c.widgetWantsFocusNow(w)
                w.setSelectionRange(i,j,insert=j)
            if trace: g.trace(i,j,word,p.h)
            return i,j,p,word
        else:
            # End of the body text.
            p.moveToThreadNext()
            if not p: break
            self.workCtrl.delete(0,'end')
            self.workCtrl.insert(0,p.b)
            for w in (self.workCtrl,c.frame.body.wrapper):
                c.widgetWantsFocusNow(w)
                w.setSelectionRange(0,0,insert=0)
            if trace: g.trace(0,0,'-->',p.h)
    return None,None,None,None
#@+node:ekr.20051025121408: *7* hide
def hide (self,event=None):

    self.c.frame.log.selectTab('Log')
#@+node:ekr.20051025071455.41: *7* ignore
def ignore(self,event=None):
    """Ignore the incorrect word for the duration of this spell check session."""
    if self.loaded:
        w = self.currentWord
        if w:
            self.spellController.ignore(w)
            self.tab.onFindButton()
#@+node:ekr.20100904095239.5914: *5* class EnchantClass
class EnchantClass:

    """A wrapper class for PyEnchant spell checker"""

    @others
#@+node:ekr.20100904095239.5916: *6*  __init__ (EnchantClass)
def __init__ (self,c):

    """Ctor for the EnchantClass class."""

    self.c = c
    language = g.toUnicode(c.config.getString('enchant_language'))
    # Set the base language
    if language and not enchant.dict_exists(language):
        g.warning('Invalid language code for Enchant',repr(language))
        g.es_print('Using "en_US" instead')
        language = 'en_US'
    # Compute fn, the full path to the local dictionary.
    fn = g.os_path_finalize(
        c.config.getString('enchant_local_dictionary') or
        os.path.join(g.app.loadDir,"..","plugins",'spellpyx.txt'))
    self.open_dict(fn,language)
#@+node:ekr.20130116142831.10185: *6* clean_dict
def clean_dict (self,fn):
    
    if g.os_path_exists(fn):
        f = open(fn,mode='rb')
        s = f.read()
        f.close()
        # Blanks lines cause troubles.
        s2 = s.replace(b'\r',b'').replace(b'\n\n',b'\n')
        if s2.startswith(b'\n'): s2 = s2[1:]
        if s != s2:
            g.es_print('cleaning',fn)
            f = open(fn,mode='wb')
            f.write(s2)
            f.close()
#@+node:ekr.20130915181927.11293: *6* create
def create (self,fn):
    
    '''Create the given file with empty contents.'''
    try:
        f = open(fn,mode='wb')
        f.close()
        g.note('created: %s' % (fn))
    except IOError:
        g.error('can not create: %s' % (fn))
    except Exception:
        g.error('unexpected error creating: %s' % (fn))
        g.es_exception()
#@+node:ekr.20100904095239.5927: *6* add
def add (self,word):

    '''Add a word to the user dictionary.'''

    self.d.add(word)
#@+node:ekr.20100904095239.5928: *6* ignore
def ignore (self,word):

    self.d.add_to_session(word)
#@+node:ekr.20130915181927.11294: *6* open_dict
def open_dict(self,fn,language):
    
    '''Open or create the dict with the given fn.'''
    if not fn or not language:
        return
    if not g.os_path_exists(fn):
        # Fix bug 1175013: leo/plugins/spellpyx.txt is both source controlled and customized.
        self.create(fn)
    if g.os_path_exists(fn):
        # Merge the local and global dictionaries.
        try:
            self.clean_dict(fn)
            self.d = enchant.DictWithPWL(language,fn)
        except Exception:
            g.es_exception()
            g.error('not a valid dictionary file',fn)
            self.d = enchant.Dict(language)
    else:
        # A fallback.  Unlikely to happen.
        self.d = enchant.Dict(language)
#@+node:ekr.20100904095239.5920: *6* processWord
def processWord(self, word):

    """Check the word. Return None if the word is properly spelled.
    Otherwise, return a list of alternatives."""

    d = self.d 

    if not d:
        return None
    elif d.check(word):
        return None
    else:
        return d.suggest(word)
#@+node:ekr.20050920084036.190: *3* COPY 5.5: MacroCommandsClass
class MacroCommandsClass (BaseEditCommandsClass):

    '''A class for recording, playing back, saving and restoring keyboard macros.
    '''

    @others
#@+node:ekr.20050920084036.191: *4*  ctor (MacroCommandsClass)
def __init__ (self,c):

    BaseEditCommandsClass.__init__(self,c) # init the base class.

    self.lastMacro = None
    self.macros = []
    self.macro = []
    self.namedMacros = {}

    # Important: we must not interfere with k.state in startRecordingMacro!
    self.recordingMacro = False
#@+node:ekr.20050920084036.192: *4*  getPublicCommands
def getPublicCommands (self):

    return {
        'macro-call':           self.callNamedMacro,
        'macro-call-last':      self.callLastMacro,
        'macro-end-recording':  self.endMacro,
        'macro-load-all':       self.loadMacros,
        'macro-name-last':      self.nameLastMacro,
        'macro-print-all':      self.printMacros,
        'macro-print-last':     self.printLastMacro,
        'macro-save-all':       self.saveMacros,
        'macro-start-recording':self.startRecordingMacro,
    }
#@+node:ekr.20050920084036.202: *4* callLastMacro
# Called from universal-command.

def callLastMacro (self,event=None):

    '''Call the last recorded keyboard macro.'''

    # g.trace(self.lastMacro)

    if self.lastMacro:
        self.executeMacro(self.lastMacro)
#@+node:ekr.20050920084036.194: *4* callNamedMacro
def callNamedMacro (self,event):

    '''Prompts for a macro name, then executes it.'''

    k = self.k ; tag = 'macro-name'
    state = k.getState(tag)
    prompt = 'Call macro named: '

    if state == 0:
        k.setLabelBlue(prompt)
        k.getArg(event,tag,1,self.callNamedMacro)
    else:
        macro = self.namedMacros.get(k.arg)
        # Must do this first!
        k.clearState()
        if macro:
            self.executeMacro(macro)
        else:
            g.es('no macro named %s' % k.arg)
        k.resetLabel()

#@+node:ekr.20050920085536.15: *4* completeMacroDef
# Called from loadFile and nameLastMacro.

def completeMacroDef (self,name,macro):

    '''Add the macro to the list of macros,
    and add the macro's name to c.commandsDict.
    '''

    trace = False and not g.unitTesting
    k= self ; c = k.c

    if trace:
        g.trace('macro::%s' % (name))
        for event in macro:
            g.trace(event.stroke)

    def func (event,macro=macro):
        return self.executeMacro(macro)

    if name in c.commandsDict:
        g.es_print('over-riding command: %s' % (name))
    else:
        g.es_print('loaded: %s' % (name))

    c.commandsDict [name] = func
    self.namedMacros [name] = macro
#@+node:ekr.20050920084036.206: *4* endMacro
def endMacro (self,event=None):
    '''Stops recording a macro.'''
    k = self.k
    self.recordingMacro = False
        # Tell k.masterKeyHandler and k.masterCommandHandler we are done.
    if self.macro:
        # self.macro = self.macro [: -4]
        self.macros.insert(0,self.macro)
        self.lastMacro = self.macro[:]
        self.macro = []
        k.setLabelBlue('Keyboard macro defined, not named')
        # g.es('Keyboard macro defined, not named')
    else:
        k.setLabelBlue('Empty keyboard macro')
        # g.es('Empty keyboard macro')
#@+node:ekr.20050920084036.203: *4* executeMacro
def executeMacro (self,macro):

    trace = False and not g.unitTesting
    c = self.c ; k = self.k

    c.bodyWantsFocus()

    for event in macro:
        if trace: g.trace(repr(event))
        k.masterKeyHandler(event)
#@+node:ekr.20110606152005.16785: *4* getMacrosNode
def getMacrosNode (self):

    '''Return the position of the @macros node.'''

    c = self.c

    for p in c.all_unique_positions():
        if p.h == '@macros':
            return p

    # Not found.
    for p in c.all_unique_positions():
        if p.h == '@settings':
            # Create as the last child of the @settings node.
            p2 = p.insertAsLastChild()
            break
    else:
        # Create as the root node.
        oldRoot = c.rootPosition()
        p2 = oldRoot.insertAfter()
        p2.moveToRoot(oldRoot)

    c.setHeadString(p2,'@macros')
    g.es_print('Created: %s' % p2.h)
    c.redraw()
    return p2
#@+node:ekr.20110606152005.16788: *4* getWidgetName
def getWidgetName(self,obj):

    if not obj:
        return ''
    if hasattr(obj,'objectName'):
        return obj.objectName()
    if hasattr(obj,'widget'):
        if hasattr(obj.widget,'objectName'):
            return obj.widget.objectName()
    return ''
#@+node:ekr.20110606152005.16787: *4* loadMacros
def loadMacros (self,event=None):
    '''Load macros from the @macros node.'''
    trace = False and not g.unitTesting
    c = self.c
    create_event = g.app.gui.create_key_event
    p = self.getMacrosNode()

    def oops(message):
        g.trace(message)

    lines = g.splitLines(p.b)
    i = 0
    macro = [] ; name = None
    while i < len(lines):
        progress = i
        s = lines[i].strip()
        i += 1
        if s.startswith('::') and s.endswith('::'):
            name = s[2:-2]
            if name:
                macro = []
                while i < len(lines):
                    s = lines[i].strip()
                    if trace: g.trace(repr(name),repr(s))
                    if s:
                        stroke=s
                        char = c.k.stroke2char(stroke)
                        w = c.frame.body.wrapper
                        macro.append(create_event(c,char,stroke,w))
                        i += 1
                    else: break
                # Create the entries.
                if macro:
                    self.completeMacroDef(name,macro)
                    macro = [] ; name = None
                else:
                    oops('empty expansion for %s' % (name))
        elif s:
            if s.startswith('#') or s.startswith('@'):
                pass
            else:
                oops('ignoring line: %s' % (repr(s)))
        else: pass
        assert progress < i
    # finish of the last macro.
    if macro:
        self.completeMacroDef(name,macro)
#@+node:ekr.20050920084036.198: *4* nameLastMacro
def nameLastMacro (self,event):

    '''Prompts for the name to be given to the last recorded macro.'''

    k = self.k ; state = k.getState('name-macro')

    if state == 0:
        k.setLabelBlue('Name of macro: ')
        k.getArg(event,'name-macro',1,self.nameLastMacro)
    else:
        k.clearState()
        name = k.arg
        self.completeMacroDef(name,self.lastMacro)
        k.setLabelGrey('Macro defined: %s' % name)
#@+node:ekr.20090201152408.1: *4* printMacros & printLastMacro
def printMacros (self,event=None):

    '''Prints the name and definition of all named macros.'''

    names = list(self.namedMacros.keys())

    if names:
        names.sort()
        print('macros',names)
        # g.es('\n'.join(names),tabName='Macros')
    else:
        g.es('no macros')

def printLastMacro (self,event=None):

    '''Print the last (unnamed) macro.'''

    if self.lastMacro:
        for event in self.lastMacro:
            g.es(repr(event.stroke))
#@+node:ekr.20050920084036.199: *4* saveMacros
def saveMacros (self,event=None):

    '''Store macros in the @macros node..'''

    p = self.getMacrosNode()
    result = []
    # g.trace(list(self.namedMacros.keys()))
    for name in self.namedMacros:
        macro = self.namedMacros.get(name)
        result.append('::%s::' % (name))
        for event in macro:
            if 0:
                w_name = self.getWidgetName(event.w)
                result.append('%s::%s::%s' % (repr(event.char),event.stroke,w_name))
            result.append(event.stroke)
        result.append('') # Blank line terminates

    p.b = '\n'.join(result)

#@+node:ekr.20050920084036.204: *4* startRecordingMacro
def startRecordingMacro (self,event):

    '''Start recording or continue to record a macro.'''

    trace = False and not g.unitTesting
    k = self.k

    if event:
        if self.recordingMacro:
            if trace: g.trace('stroke',event.stroke)
            self.macro.append(event)
        else:
            self.recordingMacro = True
            k.setLabelBlue('Recording macro. ctrl-g to end...',protect=True)
            # g.es('Recording macro. ctrl-g to end...')
    else:
        g.trace('can not happen: no event')
#@+node:ekr.20190506094028.1: ** Demo stuff
#@+node:ekr.20190506094028.2: *3* @@button demo1 @key=Ctrl-8
g.cls()
if c.isChanged(): c.save()
import imp
import leo.plugins.demo as demo
imp.reload(demo)
<< class MyDemo >>
h = 'demo1-commands'
button_p = g.findNodeAnywhere(c, '@button demo1 @key=Ctrl-8')
commands = g.findNodeInTree(c, button_p, h)
if commands:
    MyDemo(c).start(commands)
else:
    print('not found', h, c.p.h)
#@+node:ekr.20190506094028.3: *4* << class MyDemo >>
class MyDemo(demo.Demo):
    
    @others
    
#@+node:ekr.20190506094028.4: *5* setup
def setup(self, p):
    
    c = self.c
    self.delta = 0
    self.clear_log()
    p = g.findNodeAnywhere(c, 'Demo area')
    if p:
        c.selectPosition(p)
#@+node:ekr.20190506094028.5: *5* teardown
def teardown(self):
    
    c = self.c
    if self.delta:
        self.set_text_delta(-self.delta)
    p = g.findNodeAnywhere(c, 'Demo area')
    if p:
        c.selectPosition(p)
        next = p.next()
        if next and next.h == 'This is a test':
            c.selectPosition(next)
            next.doDelete()
            c.selectPosition(p)
            c.setChanged(False)
            c.redraw()
    

#@+node:ekr.20190506094028.6: *4* demo1-commands
print('demo1-commands')
# c.contractAllHeadlines()
#@+node:ekr.20190506094028.7: *5* @ignore-tree
#@+node:ekr.20190506094028.8: *6* set_text_delta
print('increasing text size by 10')
demo.delta = 10
demo.set_text_delta(demo.delta)
#@+node:ekr.20190506094028.9: *6* undo
undo_type = c.undoer.undoType
if undo_type == 'Insert Node':
    c.undoer.undo()
#@+node:ekr.20190506094028.10: *6* caption
demo.caption('My Caption', 'body')
#@+node:ekr.20190506094028.11: *6* @image
demo.delete_widgets()
fn = 'SplashScreen.ico'
demo.image('body', fn, center=True, height=None, width=None)
#@+node:ekr.20190506094028.12: *6* open menu
demo.delete_widgets()
demo.open_menu('Import')
#@+node:ekr.20190506094028.13: *6* close menu
demo.dismiss_menu_bar()
#@+node:ekr.20190506094028.14: *6* Alt-X insert-node
demo.key('Alt+x') # Not the same as Alt-X
demo.keys('insert-node')
# demo.wait(0.8)
# demo.key('\n') # Works.
#@+node:ekr.20190506094028.15: *6* Return
demo.key('\n')
#@+node:ekr.20190506094028.16: *5* headline
c.k.simulateCommand('insert-node')
demo.head_keys('This is a test')
#@+node:ekr.20190506094028.17: *3* Test: import c:\test\demo-it.el
g.cls()
import imp
import leo.plugins.importers.linescanner as linescanner
import leo.plugins.importers.elisp as elisp
imp.reload(linescanner)
imp.reload(elisp)
x = elisp.Elisp_Importer(c.importCommands, atAuto=False)
with open('c:/test/demo-it.el') as f:
    s = f.read()
parent = p.next()
assert parent.h == 'demo.el', parent.h
parent.b = ''
parent.deleteAllChildren()
try:
    x.run(s, parent)
except Exception:
    g.es_exception()
parent.expand()
c.selectPosition(parent)
c.redraw()
# g.printList(g.splitLines(s))
#@+node:ekr.20190506094028.18: *3* demo.image & helper
def image(self, fn, center=None, height=None, pane=None, width=None):
    '''Put an image in the indicated pane.'''
    parent = self.pane_widget(pane or 'body')
    if parent:
        w = QtWidgets.QLabel('label', parent)
        fn = self.resolve_icon_fn(fn)
        if not fn: return None
        pixmap = QtGui.QPixmap(fn)
        if not pixmap:
            return g.trace('Not a pixmap: %s' % (fn))
        if height:
            pixmap = pixmap.scaledToHeight(height)
        if width:
            pixmap = pixmap.scaledToWidth(width)
        w.setPixmap(pixmap)
        if center:
            g_w = w.geometry()
            g_p = parent.geometry()
            dx = (g_p.width() - g_w.width()) / 2
            w.move(g_w.x() + dx, g_w.y() + 10)
        w.show()
        self.widgets.append(w)
        return w
    else:
        g.trace('bad pane: %s' % (pane))
        return None
#@+node:ekr.20190506094028.19: *4* demo.resolve_icon_fn
def resolve_icon_fn(self, fn):
    '''Resolve fn relative to the Icons directory.'''
    dir_ = g.os_path_finalize_join(g.app.loadDir, '..', 'Icons')
    path = g.os_path_finalize_join(dir_, fn)
    if g.os_path_exists(path):
        return path
    else:
        g.trace('does not exist: %s' % (path))
        return None
#@+node:ekr.20190506094028.20: *3* demo.caption & body, log, tree
def caption(self, s, pane): # To do: center option.
    '''Pop up a QPlainTextEdit in the indicated pane.'''
    parent = self.pane_widget(pane)
    if parent:
        s = s.rstrip()
        if s and s[-1].isalpha(): s = s + '.'
        w = QtWidgets.QPlainTextEdit(s, parent)
        w.setObjectName('screencastcaption')
        self.widgets.append(w)
        w2 = self.pane_widget(pane)
        geom = w2.geometry()
        w.resize(geom.width(), min(150, geom.height() / 2))
        off = QtCore.Qt.ScrollBarAlwaysOff
        w.setHorizontalScrollBarPolicy(off)
        w.setVerticalScrollBarPolicy(off)
        w.show()
        return w
    else:
        g.trace('bad pane: %s' % (pane))
        return None

def body(self, s):
    return self.caption(s, 'body')

def log(self, s):
    return self.caption(s, 'log')

def tree(self, s):
    return self.caption(s, 'tree')
#@+node:ekr.20190506094028.21: *3* demo.body, log, tree
def body(self, s):
    return TextEdit(s, 'body')

def log(self, s):
    return TextEdit(s, 'log')

def tree(self, s):
    return TextEdit(s, 'tree')
#@+node:ekr.20190506094028.22: *3* Demo area
@language python

# A python comment.
#@+node:ekr.20190508062044.1: ** Do not delete
#@+node:ekr.20180826065640.1: *3* vr.embed_pyplot_widget (not used)
def embed_pyplot_widget(self):

    pc = self
    c = pc.c
    # Careful: we may be unit testing.
    splitter = c.free_layout.get_top_splitter()
    if not splitter:
        return
    if not pc.pyplot_canvas:

        # TODO Create the widgets.
        w = None
        ### Ref
        # pc.gs = QtWidgets.QGraphicsScene(splitter)
        # pc.gv = QtWidgets.QGraphicsView(pc.gs)
        # w = pc.gv.viewport() # A QWidget
        # Embed the widgets.
        pc.pyplot_canvas = w

        def delete_callback():
            pc.pyplot_canvas.deleteLater()
            pc.pyplot_canvas = None

    if pc.pyplot_canvas:
        pc.embed_widget(w, delete_callback=delete_callback)
#@+node:ekr.20150521114057.1: *3* test_beautifier (prints stats)
def test_beautifier(c, h, p, settings):
    '''Test Leo's beautifier code'''
    if not p:
        g.trace('not found: %s' % h)
        return None
    s = g.getScript(c, p,
            useSelectedText=False,
            forcePythonSentinels=True,
            useSentinels=False)
    g.trace(h.strip())
    t1 = time.time()
    s1 = g.toEncodedString(s)
    node1 = ast.parse(s1, filename='before', mode='exec')
    t2 = time.time()
    readlines = g.ReadLinesClass(s).next
    tokens = list(tokenize.generate_tokens(readlines))
    t3 = time.time()
    beautifier = PythonTokenBeautifier(c)
    keep_blank_lines = settings.get('tidy-keep-blank-lines')
    if keep_blank_lines is not None:
        beautifier.delete_blank_lines = not keep_blank_lines
    s2 = beautifier.run(tokens)
    t4 = time.time()
    try:
        s2_e = g.toEncodedString(s2)
        node2 = ast.parse(s2_e, filename='before', mode='exec')
        ok = compare_ast(node1, node2)
    except Exception:
        g.es_exception()
        ok = False
    t5 = time.time()
    #  Update the stats
    beautifier.n_input_tokens += len(tokens)
    beautifier.n_output_tokens += len(beautifier.code_list)
    beautifier.n_strings += len(s2)
    beautifier.parse_time += (t2 - t1)
    beautifier.tokenize_time += (t3 - t2)
    beautifier.beautify_time += (t4 - t3)
    beautifier.check_time += (t5 - t4)
    beautifier.total_time += (t5 - t1)
    if settings.get('input_string'):
        print('==================== input_string')
        for i, z in enumerate(g.splitLines(s)):
            print('%4s %s' % (i + 1, z.rstrip()))
    if settings.get('input_lines'):
        print('==================== input_lines')
        dump_tokens(tokens, verbose=False)
    if settings.get('input_tokens'):
        print('==================== input_tokens')
        dump_tokens(tokens, verbose=True)
    if settings.get('output_tokens'):
        print('==================== code_list')
        for i, z in enumerate(beautifier.code_list):
            print('%4s %s' % (i, z))
    if settings.get('output_string'):
        print('==================== output_string')
        for i, z in enumerate(g.splitLines(s2)):
            if z == '\n':
                print('%4s' % (i + 1))
            elif z.rstrip():
                print('%4s %s' % (i + 1, z.rstrip()))
            else:
                print('%4s %r' % (i + 1, str(z)))
    if settings.get('stats'):
        beautifier.print_stats()
    if not ok:
        print('*************** fail: %s ***************' % (h))
    return beautifier
        # For statistics.
#@+node:ekr.20191229062845.1: ** From leoAst.py
#@+node:ekr.20201017155356.1: *3*  ----- Support for Python 3.8 type nodes
@language rest
@wrap

For more details, see:
https://github.com/leo-editor/leo-editor/issues/

I'll support these nodes only if someone complains.

@language python
#@+node:ekr.20201017072904.1: *4* tog: Type Expressions (Python 3.8+) (not used)
# Type expressions are *comments*.
#@+node:ekr.20201017072929.1: *5* tog.FunctionType
# FunctionType(expr* argtypes, expr returns)

def do_FunctionType(self, node):

    argtypes = getattr(node, 'argtypes', [])
    for z in argtypes:
        yield from self.gen(z)
    yield from self.gen(node.returns)
#@+node:ekr.20201017072932.1: *5* tog.NamedExpr
# NamedExpr(expr target, expr value)

def do_NamedExpr(self, node):
    
    yield from self.gen(node.target)
    yield from self.gen(node.value)
#@+node:ekr.20201017072932.2: *5* tog.TypeIgnore
# type_ignore = TypeIgnore(int lineno, string tag)

def do_TypeIgnore(self, node):
    
    yield from self.gen(node.lineno)
    yield from self.gen(node.tag)
#@+node:ekr.20191027075648.1: *4* function: parse_ast
def parse_ast(s):
    """
    Parse string s, catching & reporting all exceptions.
    Return the ast node, or None.
    """

    def oops(message):
        print('')
        print(f"parse_ast: {message}")
        g.printObj(s)
        print('')

    try:
        s1 = g.toEncodedString(s)
        tree = ast.parse(s1, filename='before', mode='exec')
        return tree
    except IndentationError:
        oops('Indentation Error')
    except SyntaxError:
        oops('Syntax Error')
    except Exception:
        oops('Unexpected Exception')
        g.es_exception()
    return None
#@+node:ekr.20191116061253.1: *3*  Generator tests
@nosearch
#@+node:ekr.20191113050247.1: *4* test: dump a generator
def seq():
    """A test generator."""
    for i in range(5):
        yield i

def dump(it, tag):
    """A generator that dumps another generator."""
    aList = list(it)
    g.printObj(aList, tag=tag)
    for z in aList:
        yield z

for i in dump(seq(), tag='seq'):
    print(i)

print('done')
#@+node:ekr.20191119024754.1: *4* test: zip_longest + enumerate
import itertools

list_a = ['a1', 'a2']
list_b = ['b1', 'b2', 'b3']

it = itertools.zip_longest(list_a, list_b, fillvalue='missing')
for i, (a, b) in enumerate(it):
    print(i, a, b)
#@+node:ekr.20191123085218.1: *4* test: next(generator) & test for generator/iterable
g.cls()
from collections.abc import Iterable 
import itertools
import types

gen = (z for z in range(5))
g1, g2 = itertools.tee(gen, 2)
for x in gen, g1, g2:
    print(f"     types.GeneratorType: {isinstance(x, types.GeneratorType)!s:>5} {x}")
for x in gen, g1, g2:
    print(f"collections.abc.Iterable: {isinstance(x, Iterable)!s:>5} {x}")
for z in g1:
    print(z)
print('done 1')
try:
    while True:
        print(next(g2))
except StopIteration:
    print('done 2')
#@+node:ekr.20191123163549.1: *4* test: exhaust generator w/o generating data
g.cls()
gen = (z for z in range(5))
try:
    while True:
        next(gen)
except StopIteration:
    print('done 2')
#@+node:ekr.20200729081252.1: *3* FullTraverser classes
#@+node:ekr.20141012064706.18399: *4* class AstFormatter
class AstFormatter:
    """
    A class to recreate source code from an AST.

    This does not have to be perfect, but it should be close.

    Also supports optional annotations such as line numbers, file names, etc.
    """
    # No ctor.
    # pylint: disable=consider-using-enumerate

    in_expr = False
    level = 0

    @others
#@+node:ekr.20141012064706.18402: *5* f.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    self.level = level
    val = self.visit(node, *args, **keys)
    return val.rstrip() if val else ''
#@+node:ekr.20141012064706.18403: *5* f.visit
def visit(self, node, *args, **keys):
    """Return the formatted version of an Ast node, or list of Ast nodes."""

    if isinstance(node, (list, tuple)):
        return ','.join([self.visit(z) for z in node])
    if node is None:
        return 'None'
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    s = method(node, *args, **keys)
    assert isinstance(s, str), type(s)
    return s
#@+node:ekr.20141012064706.18469: *5* f.indent
def indent(self, s):
    return f'%s%s' % (' ' * 4 * self.level, s)
#@+node:ekr.20141012064706.18404: *5* f: Contexts
#@+node:ekr.20141012064706.18405: *6* f.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, print_body=True):

    result = []
    name = node.name  # Only a plain string is valid.
    bases = [self.visit(z) for z in node.bases] if node.bases else []
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            bases.append(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
    if getattr(node, 'starargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.starargs))
    if getattr(node, 'kwargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.kwargs))
    if bases:
        result.append(self.indent(f'class %s(%s):\n' % (name, ','.join(bases))))
    else:
        result.append(self.indent(f'class %s:\n' % name))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18406: *6* f.FunctionDef & AsyncFunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False, print_body=True):
    """Format a FunctionDef node."""
    result = []
    if node.decorator_list:
        for z in node.decorator_list:
            result.append(f'@%s\n' % self.visit(z))
    name = node.name  # Only a plain string is valid.
    args = self.visit(node.args) if node.args else ''
    asynch_prefix = 'asynch ' if async_flag else ''
    if getattr(node, 'returns', None):  # Python 3.
        returns = self.visit(node.returns)
        result.append(self.indent(f'%sdef %s(%s): -> %s\n' % (
            asynch_prefix, name, args, returns)))
    else:
        result.append(self.indent(f'%sdef %s(%s):\n' % (
            asynch_prefix, name, args)))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFunctionDef(self, node):
    return self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20141012064706.18407: *6* f.Interactive
def do_Interactive(self, node):
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18408: *6* f.Module
def do_Module(self, node):
    assert 'body' in node._fields
    result = ''.join([self.visit(z) for z in node.body])
    return result
#@+node:ekr.20141012064706.18409: *6* f.Lambda
def do_Lambda(self, node):
    return self.indent(f'lambda %s: %s' % (
        self.visit(node.args),
        self.visit(node.body)))
#@+node:ekr.20141012064706.18410: *5* f: Expressions
#@+node:ekr.20141012064706.18411: *6* f.Expr
def do_Expr(self, node):
    """An outer expression: must be indented."""
    assert not self.in_expr
    self.in_expr = True
    value = self.visit(node.value)
    self.in_expr = False
    return self.indent(f'%s\n' % value)
#@+node:ekr.20141012064706.18412: *6* f.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s\n' % self.visit(node.body)
#@+node:ekr.20141012064706.18413: *6* f.GeneratorExp
def do_GeneratorExp(self, node):
    elt = self.visit(node.elt) or ''
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'<gen %s for %s>' % (elt, ','.join(gens))
#@+node:ekr.20141012064706.18414: *6* f.ctx nodes
def do_AugLoad(self, node):
    return 'AugLoad'

def do_Del(self, node):
    return 'Del'

def do_Load(self, node):
    return 'Load'

def do_Param(self, node):
    return 'Param'

def do_Store(self, node):
    return 'Store'
#@+node:ekr.20141012064706.18415: *5* f: Operands
#@+node:ekr.20141012064706.18416: *6* f.arguments
# 2: arguments = (expr* args, identifier? vararg, identifier?
#                arg? kwarg, expr* defaults)
# 3: arguments = (arg*  args, arg? vararg,
#                arg* kwonlyargs, expr* kw_defaults,
#                arg? kwarg, expr* defaults)

def do_arguments(self, node):
    """Format the arguments node."""
    kind = node.__class__.__name__
    assert kind == 'arguments', kind
    args = [self.visit(z) for z in node.args]
    defaults = [self.visit(z) for z in node.defaults]
    args2 = []
    n_plain = len(args) - len(defaults)
    for i in range(len(node.args)):
        if i < n_plain:
            args2.append(args[i])
        else:
            args2.append(f'%s=%s' % (args[i], defaults[i-n_plain]))
    # Add the vararg and kwarg expressions.
    vararg = getattr(node, 'vararg', None)
    if vararg: args2.append('*'+self.visit(vararg))
    kwarg = getattr(node, 'kwarg', None)
    if kwarg: args2.append(f'**'+self.visit(kwarg))
    return ','.join(args2)
#@+node:ekr.20141012064706.18417: *6* f.arg (Python3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        return self.visit(node.annotation)
    return node.arg
#@+node:ekr.20141012064706.18418: *6* f.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    return f'%s.%s' % (
        self.visit(node.value),
        node.attr)  # Don't visit node.attr: it is always a string.
#@+node:ekr.20141012064706.18419: *6* f.Bytes
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20141012064706.18420: *6* f.Call & f.keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    func = self.visit(node.func)
    args = [self.visit(z) for z in node.args]
    for z in node.keywords:
        # Calls f.do_keyword.
        args.append(self.visit(z))
    if getattr(node, 'starargs', None):
        args.append(f'*%s' % (self.visit(node.starargs)))
    if getattr(node, 'kwargs', None):
        args.append(f'**%s' % (self.visit(node.kwargs)))
    args = [z for z in args if z]  # Kludge: Defensive coding.
    s = f'%s(%s)' % (func, ','.join(args))
    return s if self.in_expr else self.indent(s+'\n')
        # 2017/12/15.
#@+node:ekr.20141012064706.18421: *7* f.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    value = self.visit(node.value)
    # This is a keyword *arg*, not a Python keyword!
    return f'%s=%s' % (node.arg, value)
#@+node:ekr.20141012064706.18422: *6* f.comprehension
def do_comprehension(self, node):
    result = []
    name = self.visit(node.target)  # A name.
    it = self.visit(node.iter)  # An attribute.
    result.append(f'%s in %s' % (name, it))
    ifs = [self.visit(z) for z in node.ifs]
    if ifs:
        result.append(f' if %s' % (''.join(ifs)))
    return ''.join(result)
#@+node:ekr.20170721073056.1: *6* f.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20141012064706.18423: *6* f.Dict
def do_Dict(self, node):
    result = []
    keys = [self.visit(z) for z in node.keys]
    values = [self.visit(z) for z in node.values]
    if len(keys) == len(values):
        result.append('{\n' if keys else '{')
        items = []
        for i in range(len(keys)):
            items.append(f'  %s:%s' % (keys[i], values[i]))
        result.append(',\n'.join(items))
        result.append('\n}' if keys else '}')
    else:
        print(
            f"Error: f.Dict: len(keys) != len(values)\n"
            f"keys: {repr(keys)}\nvals: {repr(values)}")
    return ''.join(result)
#@+node:ekr.20160523101618.1: *6* f.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    key = self.visit(node.key)
    value = self.visit(node.value)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s:%s for %s' % (key, value, ''.join(gens))
#@+node:ekr.20141012064706.18424: *6* f.Ellipsis
def do_Ellipsis(self, node):
    return '...'
#@+node:ekr.20141012064706.18425: *6* f.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20170721075130.1: *6* f.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    return f'%s%s%s' % (
        self.visit(node.value),
        self.visit(node.conversion) if node.conversion else '',
        self.visit(node.format_spec) if node.format_spec else '')
#@+node:ekr.20141012064706.18426: *6* f.Index
def do_Index(self, node):
    return self.visit(node.value)
#@+node:ekr.20170721080559.1: *6* f.JoinedStr (Python 3.6)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):

    if node.values:
        for value in node.values:
            self.visit(value)
#@+node:ekr.20141012064706.18427: *6* f.List
def do_List(self, node):
    # Not used: list context.
    # self.visit(node.ctx)
    elts = [self.visit(z) for z in node.elts]
    elts = [z for z in elts if z]  # Defensive.
    return f'[%s]' % ','.join(elts)
#@+node:ekr.20141012064706.18428: *6* f.ListComp
def do_ListComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18429: *6* f.Name & NameConstant
def do_Name(self, node):
    return node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return s
#@+node:ekr.20141012064706.18430: *6* f.Num
def do_Num(self, node):
    return repr(node.n)
#@+node:ekr.20141012064706.18431: *6* f.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20160523101929.1: *6* f.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523102226.1: *6* f.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18432: *6* f.Slice
def do_Slice(self, node):
    lower, upper, step = '', '', ''
    if getattr(node, 'lower', None) is not None:
        lower = self.visit(node.lower)
    if getattr(node, 'upper', None) is not None:
        upper = self.visit(node.upper)
    if getattr(node, 'step', None) is not None:
        step = self.visit(node.step)
    if step:
        return f'%s:%s:%s' % (lower, upper, step)
    return f'%s:%s' % (lower, upper)
#@+node:ekr.20141012064706.18433: *6* f.Str
def do_Str(self, node):
    """This represents a string constant."""
    return repr(node.s)
#@+node:ekr.20141012064706.18434: *6* f.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    value = self.visit(node.value)
    the_slice = self.visit(node.slice)
    return f'%s[%s]' % (value, the_slice)
#@+node:ekr.20141012064706.18435: *6* f.Tuple
def do_Tuple(self, node):
    elts = [self.visit(z) for z in node.elts]
    return f'(%s)' % ','.join(elts)
#@+node:ekr.20141012064706.18436: *5* f: Operators
#@+node:ekr.20141012064706.18437: *6* f.BinOp
def do_BinOp(self, node):
    return f'%s%s%s' % (
        self.visit(node.left),
        op_name(node.op),
        self.visit(node.right))
#@+node:ekr.20141012064706.18438: *6* f.BoolOp
def do_BoolOp(self, node):
    op_name_ = op_name(node.op)
    values = [self.visit(z).strip() for z in node.values]
    return op_name_.join(values)
#@+node:ekr.20141012064706.18439: *6* f.Compare
def do_Compare(self, node):
    result = []
    lt = self.visit(node.left)
    # ops   = [self.visit(z) for z in node.ops]
    ops = [op_name(z) for z in node.ops]
    comps = [self.visit(z) for z in node.comparators]
    result.append(lt)
    assert len(ops) == len(comps), repr(node)
    for i in range(len(ops)):
        result.append(f'%s%s' % (ops[i], comps[i]))
    return ''.join(result)
#@+node:ekr.20141012064706.18440: *6* f.UnaryOp
def do_UnaryOp(self, node):
    return f'%s%s' % (
        op_name(node.op),
        self.visit(node.operand))
#@+node:ekr.20141012064706.18441: *6* f.ifExp (ternary operator)
def do_IfExp(self, node):
    return f'%s if %s else %s ' % (
        self.visit(node.body),
        self.visit(node.test),
        self.visit(node.orelse))
#@+node:ekr.20141012064706.18442: *5* f: Statements
#@+node:ekr.20170721074105.1: *6* f.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    return self.indent(f'%s:%s=%s\n' % (
        self.visit(node.target),
        self.visit(node.annotation),
        self.visit(node.value),
    ))
#@+node:ekr.20141012064706.18443: *6* f.Assert
def do_Assert(self, node):
    test = self.visit(node.test)
    if getattr(node, 'msg', None):
        message = self.visit(node.msg)
        return self.indent(f'assert %s, %s' % (test, message))
    return self.indent(f'assert %s' % test)
#@+node:ekr.20141012064706.18444: *6* f.Assign
def do_Assign(self, node):
    return self.indent(f'%s=%s\n' % (
        '='.join([self.visit(z) for z in node.targets]),
        self.visit(node.value)))
#@+node:ekr.20141012064706.18445: *6* f.AugAssign
def do_AugAssign(self, node):
    return self.indent(f'%s%s=%s\n' % (
        self.visit(node.target),
        op_name(node.op),  # Bug fix: 2013/03/08.
        self.visit(node.value)))
#@+node:ekr.20160523100504.1: *6* f.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    return self.indent(f'await %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18446: *6* f.Break
def do_Break(self, node):
    return self.indent(f'break\n')
#@+node:ekr.20141012064706.18447: *6* f.Continue
def do_Continue(self, node):
    return self.indent(f'continue\n')
#@+node:ekr.20141012064706.18448: *6* f.Delete
def do_Delete(self, node):
    targets = [self.visit(z) for z in node.targets]
    return self.indent(f'del %s\n' % ','.join(targets))
#@+node:ekr.20141012064706.18449: *6* f.ExceptHandler
def do_ExceptHandler(self, node):
    
    result = []
    result.append(self.indent('except'))
    if getattr(node, 'type', None):
        result.append(f' %s' % self.visit(node.type))
    if getattr(node, 'name', None):
        if isinstance(node.name, ast.AST):
            result.append(f' as %s' % self.visit(node.name))
        else:
            result.append(f' as %s' % node.name)  # Python 3.x.
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18450: *6* f.Exec
# Python 2.x only

def do_Exec(self, node):
    body = self.visit(node.body)
    args = []  # Globals before locals.
    if getattr(node, 'globals', None):
        args.append(self.visit(node.globals))
    if getattr(node, 'locals', None):
        args.append(self.visit(node.locals))
    if args:
        return self.indent(f'exec %s in %s\n' % (
            body, ','.join(args)))
    return self.indent(f'exec {body}\n')
#@+node:ekr.20141012064706.18451: *6* f.For & AsnchFor (Python 3)
def do_For(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%sfor %s in %s:\n' % (
        'async ' if async_flag else '',
        self.visit(node.target),
        self.visit(node.iter))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFor(self, node):
    return self.do_For(node, async_flag=True)
#@+node:ekr.20141012064706.18452: *6* f.Global
def do_Global(self, node):
    return self.indent(f'global %s\n' % (
        ','.join(node.names)))
#@+node:ekr.20141012064706.18453: *6* f.If
def do_If(self, node):
    result = []
    result.append(self.indent(f'if %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent(f'else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18454: *6* f.Import & helper
def do_Import(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'import %s\n' % (
        ','.join(names)))
#@+node:ekr.20141012064706.18455: *7* f.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert ast2.__class__.__name__ == 'alias', (repr(ast2))
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20141012064706.18456: *6* f.ImportFrom
def do_ImportFrom(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'from %s import %s\n' % (
        node.module,
        ','.join(names)))
#@+node:ekr.20160317050557.2: *6* f.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    return self.indent(f'nonlocal %s\n' % ', '.join(node.names))
#@+node:ekr.20141012064706.18457: *6* f.Pass
def do_Pass(self, node):
    return self.indent('pass\n')
#@+node:ekr.20141012064706.18458: *6* f.Print
# Python 2.x only

def do_Print(self, node):
    vals = []
    for z in node.values:
        vals.append(self.visit(z))
    if getattr(node, 'dest', None):
        vals.append(f'dest=%s' % self.visit(node.dest))
    if getattr(node, 'nl', None):
        # vals.append('nl=%s' % self.visit(node.nl))
        vals.append(f'nl=%s' % node.nl)
    return self.indent(f'print(%s)\n' % (
        ','.join(vals)))
#@+node:ekr.20141012064706.18459: *6* f.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):
    args = []
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            args.append(self.visit(getattr(node, attr)))
    if args:
        return self.indent(f'raise %s\n' % (
            ','.join(args)))
    return self.indent('raise\n')
#@+node:ekr.20141012064706.18460: *6* f.Return
def do_Return(self, node):
    if node.value:
        return self.indent(f'return %s\n' % (
            self.visit(node.value)))
    return self.indent('return\n')
#@+node:ekr.20160317050557.3: *6* f.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    return '*' + self.visit(node.value)
#@+node:ekr.20141012064706.18461: *6* f.Suite
# def do_Suite(self,node):
    # for z in node.body:
        # s = self.visit(z)
#@+node:ekr.20160317050557.4: *6* f.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):  # Python 3

    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    if node.finalbody:
        result.append(self.indent('finally:\n'))
        for z in node.finalbody:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18462: *6* f.TryExcept
def do_TryExcept(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18463: *6* f.TryFinally
def do_TryFinally(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append(self.indent('finally:\n'))
    for z in node.finalbody:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18464: *6* f.While
def do_While(self, node):
    result = []
    result.append(self.indent(f'while %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18465: *6* f.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%swith ' % ('async ' if async_flag else '')))
    if getattr(node, 'context_expression', None):
        result.append(self.visit(node.context_expresssion))
    vars_list = []
    if getattr(node, 'optional_vars', None):
        try:
            for z in node.optional_vars:
                vars_list.append(self.visit(z))
        except TypeError:  # Not iterable.
            vars_list.append(self.visit(node.optional_vars))
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            result.append(self.visit(item.context_expr))
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        vars_list.append(self.visit(z))
                except TypeError:  # Not iterable.
                    vars_list.append(self.visit(item.optional_vars))
    result.append(','.join(vars_list))
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append('\n')
    return ''.join(result)

def do_AsyncWith(self, node):
    return self.do_With(node, async_flag=True)
#@+node:ekr.20141012064706.18466: *6* f.Yield
def do_Yield(self, node):
    if getattr(node, 'value', None):
        return self.indent(f'yield %s\n' % (
            self.visit(node.value)))
    return self.indent('yield\n')
#@+node:ekr.20160317050557.5: *6* f.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    return self.indent(f'yield from %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18471: *4* class AstFullTraverser
class AstFullTraverser:
    """
    A fast traverser for AST trees: it visits every node (except node.ctx fields).

    Sets .context and .parent ivars before visiting each node.
    """

    def __init__(self):
        """Ctor for AstFullTraverser class."""
        self.context = None
        self.level = 0  # The context level only.
        self.parent = None

    @others
#@+node:ekr.20141012064706.18472: *5* ft.contexts
#@+node:ekr.20141012064706.18473: *6* ft.ClassDef
# 2: ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, visit_body=True):
    old_context = self.context
    self.context = node
    self.level += 1
    for z in node.decorator_list:
        self.visit(z)
    for z in node.bases:
        self.visit(z)
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            self.visit(keyword.value)
    if getattr(node, 'starargs', None):  # Python 3
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):  # Python 3
        self.visit(node.kwargs)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context
#@+node:ekr.20141012064706.18474: *6* ft.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, visit_body=True):

    old_context = self.context
    self.context = node
    self.level += 1
    # Visit the tree in token order.
    for z in node.decorator_list:
        self.visit(z)
    assert isinstance(node.name, str)
    self.visit(node.args)
    if getattr(node, 'returns', None):  # Python 3.
        self.visit(node.returns)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context

do_AsyncFunctionDef = do_FunctionDef
#@+node:ekr.20141012064706.18475: *6* ft.Interactive
def do_Interactive(self, node):
    assert False, 'Interactive context not supported'
#@+node:ekr.20141012064706.18476: *6* ft.Lambda
# Lambda(arguments args, expr body)

def do_Lambda(self, node):
    old_context = self.context
    self.context = node
    self.visit(node.args)
    self.visit(node.body)
    self.context = old_context
#@+node:ekr.20141012064706.18477: *6* ft.Module
def do_Module(self, node):
    self.context = node
    for z in node.body:
        self.visit(z)
    self.context = None
#@+node:ekr.20141012064706.18478: *5* ft.ctx nodes
# Not used in this class, but may be called by subclasses.

def do_AugLoad(self, node):
    pass

def do_Del(self, node):
    pass

def do_Load(self, node):
    pass

def do_Param(self, node):
    pass

def do_Store(self, node):
    pass
#@+node:ekr.20171214200319.1: *5* ft.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    s = AstFormatter().format(node, level, *args, **keys)
    return s.rstrip()
#@+node:ekr.20141012064706.18480: *5* ft.operators & operands
#@+node:ekr.20160521102250.1: *6* ft.op_name
def op_name(self, node, strict=True):
    """Return the print name of an operator node."""
    name = _op_names.get(node.__class__.__name__, f'<%s>' % node.__class__.__name__)
    if strict:
        assert name, node.__class__.__name__
    return name
#@+node:ekr.20141012064706.18482: *6* ft.arguments & arg
# 2: arguments = (
# expr* args,
#   identifier? vararg,
#   identifier? kwarg,
#   expr* defaults)
# 3: arguments = (
#   arg*  args,
#   arg? vararg,
#   arg* kwonlyargs,
#   expr* kw_defaults,
#   arg? kwarg,
#   expr* defaults)

def do_arguments(self, node):

    for z in node.args:
        self.visit(z)
    if getattr(node, 'vararg', None):
        # An identifier in Python 2.
        self.visit(node.vararg)
    if getattr(node, 'kwarg', None):
        # An identifier in Python 2.
        self.visit_list(node.kwarg)
    if getattr(node, 'kwonlyargs', None):  # Python 3.
        self.visit_list(node.kwonlyargs)
    if getattr(node, 'kw_defaults', None):  # Python 3.
        self.visit_list(node.kw_defaults)
    for z in node.defaults:
        self.visit(z)

# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        self.visit(node.annotation)
#@+node:ekr.20141012064706.18483: *6* ft.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    self.visit(node.value)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18484: *6* ft.BinOp
# BinOp(expr left, operator op, expr right)

def do_BinOp(self, node):
    self.visit(node.left)
    # self.op_name(node.op)
    self.visit(node.right)
#@+node:ekr.20141012064706.18485: *6* ft.BoolOp
# BoolOp(boolop op, expr* values)

def do_BoolOp(self, node):
    for z in node.values:
        self.visit(z)
#@+node:ekr.20141012064706.18481: *6* ft.Bytes
def do_Bytes(self, node):
    pass  # Python 3.x only.
#@+node:ekr.20141012064706.18486: *6* ft.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):
    # Call the nodes in token order.
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
#@+node:ekr.20141012064706.18487: *6* ft.Compare
# Compare(expr left, cmpop* ops, expr* comparators)

def do_Compare(self, node):
    # Visit all nodes in token order.
    self.visit(node.left)
    assert len(node.ops) == len(node.comparators)
    for i in range(len(node.ops)):
        self.visit(node.ops[i])
        self.visit(node.comparators[i])
    # self.visit(node.left)
    # for z in node.comparators:
        # self.visit(z)
#@+node:ekr.20150526140323.1: *6* ft.Compare ops
# Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn

def do_Eq(self, node): pass

def do_Gt(self, node): pass

def do_GtE(self, node): pass

def do_In(self, node): pass

def do_Is(self, node): pass

def do_IsNot(self, node): pass

def do_Lt(self, node): pass

def do_LtE(self, node): pass

def do_NotEq(self, node): pass

def do_NotIn(self, node): pass
#@+node:ekr.20141012064706.18488: *6* ft.comprehension
# comprehension (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):
    self.visit(node.target)  # A name.
    self.visit(node.iter)  # An attribute.
    for z in node.ifs:
        self.visit(z)
#@+node:ekr.20170721073315.1: *6* ft.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    pass
#@+node:ekr.20141012064706.18489: *6* ft.Dict
# Dict(expr* keys, expr* values)

def do_Dict(self, node):
    # Visit all nodes in token order.
    assert len(node.keys) == len(node.values)
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.visit(node.values[i])
#@+node:ekr.20160523094910.1: *6* ft.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    # EKR: visit generators first, then value.
    for z in node.generators:
        self.visit(z)
    self.visit(node.value)
    self.visit(node.key)
#@+node:ekr.20150522081707.1: *6* ft.Ellipsis
def do_Ellipsis(self, node):
    pass
#@+node:ekr.20141012064706.18490: *6* ft.Expr
# Expr(expr value)

def do_Expr(self, node):
    self.visit(node.value)
#@+node:ekr.20141012064706.18491: *6* ft.Expression
def do_Expression(self, node):
    """An inner expression"""
    self.visit(node.body)
#@+node:ekr.20141012064706.18492: *6* ft.ExtSlice
def do_ExtSlice(self, node):
    for z in node.dims:
        self.visit(z)
#@+node:ekr.20170721075714.1: *6* ft.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
#@+node:ekr.20141012064706.18493: *6* ft.GeneratorExp
# GeneratorExp(expr elt, comprehension* generators)

def do_GeneratorExp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18494: *6* ft.ifExp (ternary operator)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):
    self.visit(node.body)
    self.visit(node.test)
    self.visit(node.orelse)
#@+node:ekr.20141012064706.18495: *6* ft.Index
def do_Index(self, node):
    self.visit(node.value)
#@+node:ekr.20170721080935.1: *6* ft.JoinedStr (Python 3.6+)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20141012064706.18496: *6* ft.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    self.visit(node.value)
#@+node:ekr.20141012064706.18497: *6* ft.List & ListComp
# List(expr* elts, expr_context ctx)

def do_List(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18498: *6* ft.Name (revise)
# Name(identifier id, expr_context ctx)

def do_Name(self, node):
    # self.visit(node.ctx)
    pass

def do_NameConstant(self, node):  # Python 3 only.
    pass
    # s = repr(node.value)
    # return 'bool' if s in ('True', 'False') else s
#@+node:ekr.20150522081736.1: *6* ft.Num
def do_Num(self, node):
    pass  # Num(object n) # a number as a PyObject.
#@+node:ekr.20141012064706.18499: *6* ft.Repr
# Python 2.x only
# Repr(expr value)

def do_Repr(self, node):
    self.visit(node.value)
#@+node:ekr.20160523094939.1: *6* ft.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)

#@+node:ekr.20160523095142.1: *6* ft.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):
    # EKR: visit generators first.
    for z in node.generators:
        self.visit(z)
    self.visit(node.elt)
#@+node:ekr.20141012064706.18500: *6* ft.Slice
def do_Slice(self, node):
    if getattr(node, 'lower', None):
        self.visit(node.lower)
    if getattr(node, 'upper', None):
        self.visit(node.upper)
    if getattr(node, 'step', None):
        self.visit(node.step)
#@+node:ekr.20150522081748.1: *6* ft.Str
def do_Str(self, node):
    pass  # represents a string constant.
#@+node:ekr.20141012064706.18501: *6* ft.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    self.visit(node.value)
    self.visit(node.slice)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18502: *6* ft.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18503: *6* ft.UnaryOp
# UnaryOp(unaryop op, expr operand)

def do_UnaryOp(self, node):
    # self.op_name(node.op)
    self.visit(node.operand)
#@+node:ekr.20141012064706.18504: *5* ft.statements
#@+node:ekr.20141012064706.18505: *6* ft.alias
# identifier name, identifier? asname)

def do_alias(self, node):
    # self.visit(node.name)
    # if getattr(node,'asname')
        # self.visit(node.asname)
    pass
#@+node:ekr.20170721074528.1: *6* ft.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    self.visit(node.target)
    self.visit(node.annotation)
    self.visit(node.value)
#@+node:ekr.20141012064706.18506: *6* ft.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):
    self.visit(node.test)
    if node.msg:
        self.visit(node.msg)
#@+node:ekr.20141012064706.18507: *6* ft.Assign
# Assign(expr* targets, expr value)

def do_Assign(self, node):
    for z in node.targets:
        self.visit(z)
    self.visit(node.value)
#@+node:ekr.20141012064706.18508: *6* ft.AugAssign
# AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    self.visit(node.target)
    self.visit(node.value)
#@+node:ekr.20141012064706.18509: *6* ft.Break
def do_Break(self, tree):
    pass
#@+node:ekr.20141012064706.18510: *6* ft.Continue
def do_Continue(self, tree):
    pass
#@+node:ekr.20141012064706.18511: *6* ft.Delete
# Delete(expr* targets)

def do_Delete(self, node):
    for z in node.targets:
        self.visit(z)
#@+node:ekr.20141012064706.18512: *6* ft.ExceptHandler
# Python 2: ExceptHandler(expr? type, expr? name, stmt* body)
# Python 3: ExceptHandler(expr? type, identifier? name, stmt* body)

def do_ExceptHandler(self, node):

    if node.type:
        self.visit(node.type)
    if node.name and isinstance(node.name, ast.Name):
        self.visit(node.name)
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18513: *6* ft.Exec
# Python 2.x only
# Exec(expr body, expr? globals, expr? locals)

def do_Exec(self, node):
    self.visit(node.body)
    if getattr(node, 'globals', None):
        self.visit(node.globals)
    if getattr(node, 'locals', None):
        self.visit(node.locals)
#@+node:ekr.20141012064706.18514: *6* ft.For & AsyncFor
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node):
    self.visit(node.target)
    self.visit(node.iter)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)

do_AsyncFor = do_For
#@+node:ekr.20141012064706.18515: *6* ft.Global
# Global(identifier* names)

def do_Global(self, node):
    pass
#@+node:ekr.20141012064706.18516: *6* ft.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node):
    self.visit(node.test)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18517: *6* ft.Import & ImportFrom
# Import(alias* names)

def do_Import(self, node):
    pass
# ImportFrom(identifier? module, alias* names, int? level)

def do_ImportFrom(self, node):
    # for z in node.names:
        # self.visit(z)
    pass
#@+node:ekr.20160317051434.2: *6* ft.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    pass
#@+node:ekr.20141012064706.18518: *6* ft.Pass
def do_Pass(self, node):
    pass
#@+node:ekr.20141012064706.18519: *6* ft.Print
# Python 2.x only
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):
    if getattr(node, 'dest', None):
        self.visit(node.dest)
    for expr in node.values:
        self.visit(expr)
#@+node:ekr.20141012064706.18520: *6* ft.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    for attr in ('exc', 'cause'):
        if getattr(node, attr, None):
            self.visit(getattr(node, attr))
#@+node:ekr.20141012064706.18521: *6* ft.Return
# Return(expr? value)

def do_Return(self, node):
    if node.value:
        self.visit(node.value)
#@+node:ekr.20160317051434.3: *6* ft.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.visit(node.value)
#@+node:ekr.20141012064706.18522: *6* ft.Try (Python 3)
# Python 3 only: Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18523: *6* ft.TryExcept
# TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)

def do_TryExcept(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18524: *6* ft.TryFinally
# TryFinally(stmt* body, stmt* finalbody)

def do_TryFinally(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18525: *6* ft.While
# While(expr test, stmt* body, stmt* orelse)

def do_While(self, node):
    self.visit(node.test)  # Bug fix: 2013/03/23.
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18526: *6* ft.With & AsyncWith
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node):
    if getattr(node, 'context_expr', None):
        self.visit(node.context_expr)
    if getattr(node, 'optional_vars', None):
        self.visit(node.optional_vars)
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        self.visit(z)
                except TypeError:  # Not iterable.
                    self.visit(item.optional_vars)
    for z in node.body:
        self.visit(z)

do_AsyncWith = do_With
#@+node:ekr.20141012064706.18527: *6* ft.Yield, YieldFrom & Await (Python 3)
# Yield(expr? value)
# Await(expr value)         Python 3 only.
# YieldFrom (expr value)    Python 3 only.

def do_Yield(self, node):
    if node.value:
        self.visit(node.value)

do_Await = do_YieldFrom = do_Yield
#@+node:ekr.20141012064706.18528: *5* ft.visit (supports before_* & after_*)
def visit(self, node):
    """Visit a *single* ast node.  Visitors are responsible for visiting children!"""
    name = node.__class__.__name__
    assert isinstance(node, ast.AST), repr(node)
    # Visit the children with the new parent.
    old_parent = self.parent
    self.parent = node
    before_method = getattr(self, 'before_'+name, None)
    if before_method:
        before_method(node)
    do_method = getattr(self, 'do_'+name, None)
    if do_method:
        val = do_method(node)
    after_method = getattr(self, 'after_'+name, None)
    if after_method:
        after_method(node)
    self.parent = old_parent
    return val

def visit_children(self, node):
    assert False, 'must visit children explicitly'
#@+node:ekr.20141012064706.18529: *5* ft.visit_list
def visit_list(self, aList):
    """Visit all ast nodes in aList or ast.node."""
    if isinstance(aList, (list, tuple)):
        for z in aList:
            self.visit(z)
        return None
    assert isinstance(aList, ast.AST), repr(aList)
    return self.visit(aList)
#@+node:ekr.20141012064706.18530: *4* class AstPatternFormatter (AstFormatter)
class AstPatternFormatter(AstFormatter):
    """
    A subclass of AstFormatter that replaces values of constants by Bool,
    Bytes, Int, Name, Num or Str.
    """
    # No ctor.
    @others
#@+node:ekr.20141012064706.18531: *5* Constants & Name
# Return generic markers allow better pattern matches.

def do_BoolOp(self, node):  # Python 2.x only.
    return 'Bool'

def do_Bytes(self, node):  # Python 3.x only.
    return 'Bytes'  # return str(node.s)

def do_Constant(self, node):  # Python 3.6+ only.
    return 'Constant'

def do_Name(self, node):
    return 'Bool' if node.id in ('True', 'False') else node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return 'Bool' if s in ('True', 'False') else s

def do_Num(self, node):
    return 'Num'  # return repr(node.n)

def do_Str(self, node):
    """This represents a string constant."""
    return 'Str'  # return repr(node.s)
#@+node:ekr.20150722204300.1: *4* class HTMLReportTraverser
class HTMLReportTraverser:
    """
    Create html reports from an AST tree.

    Inspired by Paul Boddie.

    This version writes all html to a global code list.

    At present, this code does not show comments.
    The TokenSync class is probably the best way to do this.
    """
    # To do: revise report-traverser-debug.css.
    @others
#@+node:ekr.20150722204300.2: *5* rt.__init__
def __init__(self, debug=False):
    """Ctor for the NewHTMLReportTraverser class."""
    self.code_list = []
    self.debug = debug
    self.div_stack = []
        # A check to ensure matching div/end_div.
    self.last_doc = None
    # List of divs & spans to generate...
    self.enable_list = [
        'body', 'class', 'doc', 'function',
        'keyword', 'name', 'statement'
    ]
    # Formatting stuff...
    debug_css = 'report-traverser-debug.css'
    plain_css = 'report-traverser.css'
    self.css_fn = debug_css if debug else plain_css
    self.html_footer = '\n</body>\n</html>\n'
    self.html_header = self.define_html_header()
#@+node:ekr.20150722204300.3: *6* define_html_header
def define_html_header(self):
    # Use string catenation to avoid using g.adjustTripleString.
    return (
        '<?xml version="1.0" encoding="iso-8859-15"?>\n'
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"\n'
        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n'
        '<html xmlns="http://www.w3.org/1999/xhtml">\n'
        '<head>\n'
        '  <title>%(title)s</title>\n'
        '  <link rel="stylesheet" type="text/css" href="%(css-fn)s" />\n'
        '</head>\n<body>'
    )
#@+node:ekr.20150723094359.1: *5* rt.code generators
#@+node:ekr.20150723100236.1: *6* rt.blank
def blank(self):
    """Insert a single blank."""
    self.clean(' ')
    if self.code_list[-1] not in ' \n':
        self.gen(' ')
#@+node:ekr.20150723100208.1: *6* rt.clean
def clean(self, s):
    """Remove s from the code list."""
    s2 = self.code_list[-1]
    if s2 == s:
        self.code_list.pop()
#@+node:ekr.20150723105702.1: *6* rt.colon
def colon(self):

    self.clean('\n')
    self.clean(' ')
    self.clean('\n')
    self.gen(':')
#@+node:ekr.20150723100346.1: *6* rt.comma & clean_comma
def comma(self):

    self.clean(' ')
    self.gen(', ')

def clean_comma(self):

    self.clean(', ')
#@+node:ekr.20150722204300.21: *6* rt.doc
# Called by ClassDef & FunctionDef visitors.

def doc(self, node):
    doc = ast.get_docstring(node)
    if doc:
        self.docstring(doc)
        self.last_doc = doc  # Attempt to suppress duplicate.
#@+node:ekr.20150722204300.22: *6* rt.docstring
def docstring(self, s):

    import textwrap
    self.gen("<pre class='doc'>")
    self.gen('"""')
    self.gen(self.text(textwrap.dedent(s.replace('"""', '\\"\\"\\"'))))
    self.gen('"""')
    self.gen("</pre>")
#@+node:ekr.20150722211115.1: *6* rt.gen
def gen(self, s):
    """Append s to the global code list."""
    if s:
        self.code_list.append(s)
#@+node:ekr.20150722204300.23: *6* rt.keyword (code generator)
def keyword(self, name):

    self.blank()
    self.span('keyword')
    self.gen(name)
    self.end_span('keyword')
    self.blank()
#@+node:ekr.20150722204300.24: *6* rt.name
def name(self, name):

    # Div would put each name on a separate line.
    # span messes up whitespace, for now.
    # self.span('name')
    self.gen(name)
    # self.end_span('name')
#@+node:ekr.20150723100417.1: *6* rt.newline
def newline(self):

    self.clean(' ')
    self.clean('\n')
    self.clean(' ')
    self.gen('\n')
#@+node:ekr.20150722204300.26: *6* rt.op
def op(self, op_name, leading=False, trailing=True):

    if leading:
        self.blank()
    # self.span('operation')
    # self.span('operator')
    self.gen(self.text(op_name))
    # self.end_span('operator')
    if trailing:
        self.blank()
    # self.end_span('operation')
#@+node:ekr.20160315184954.1: *6* rt.string (code generator)
def string(self, s):

    import xml.sax.saxutils as saxutils
    s = repr(s.strip().strip())
    s = saxutils.escape(s)
    self.gen(s)
#@+node:ekr.20150722204300.27: *6* rt.simple_statement
def simple_statement(self, name):

    class_name = f'%s nowrap' % name
    self.div(class_name)
    self.keyword(name)
    self.end_div(class_name)
#@+node:ekr.20150722204300.16: *5* rt.html helpers
#@+node:ekr.20150722204300.17: *6* rt.attr & text
def attr(self, s):
    return self.text(s).replace("'", "&apos;").replace('"', "&quot;")

def text(self, s):
    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
#@+node:ekr.20150722204300.18: *6* rt.br
def br(self):
    return '\n<br />'
#@+node:ekr.20150722204300.19: *6* rt.comment
def comment(self, comment):

    self.span('comment')
    self.gen('# '+comment)
    self.end_span('comment')
    self.newline()
#@+node:ekr.20150722204300.20: *6* rt.div
def div(self, class_name, extra=None, wrap=False):
    """Generate the start of a div element."""
    if class_name in self.enable_list:
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
        self.newline()
        if class_name and extra:
            self.gen(f"<div class='%s' %s>" % (full_class_name, extra))
        elif class_name:
            self.newline()
            self.gen(f"<div class='%s'>" % (full_class_name))
        else:
            assert not extra
            self.gen("<div>")
    self.div_stack.append(class_name)
#@+node:ekr.20150722222149.1: *6* rt.div_body
def div_body(self, aList):
    if aList:
        self.div_list('body', aList)
#@+node:ekr.20150722221101.1: *6* rt.div_list & div_node
def div_list(self, class_name, aList, sep=None):

    self.div(class_name)
    self.visit_list(aList, sep=sep)
    self.end_div(class_name)

def div_node(self, class_name, node):

    self.div(class_name)
    self.visit(node)
    self.end_div(class_name)
#@+node:ekr.20150723095033.1: *6* rt.end_div
def end_div(self, class_name):

    if class_name in self.enable_list:
        # self.newline()
        self.gen('</div>')
        # self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150723095004.1: *6* rt.end_span
def end_span(self, class_name):

    if class_name in self.enable_list:
        self.gen('</span>')
        self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150722221408.1: *6* rt.keyword_colon
# def keyword_colon(self, keyword):

    # self.keyword(keyword)
    # self.colon()
#@+node:ekr.20150722204300.5: *6* rt.link
def link(self, class_name, href, a_text):

    return f"<a class='%s' href='%s'>%s</a>" % (
        class_name, href, a_text)
#@+node:ekr.20150722204300.6: *6* rt.module_link
def module_link(self, module_name, classes=None):

    return self.link(
        class_name=classes or 'name',
        href=f'%s.xhtml' % module_name,
        a_text=self.text(module_name))
#@+node:ekr.20150722204300.7: *6* rt.name_link
def name_link(self, module_name, full_name, name, classes=None):

    return self.link(
        class_name=classes or "specific-ref",
        href=f'%s.xhtml#%s' % (module_name, self.attr(full_name)),
        a_text=self.text(name))
#@+node:ekr.20150722204300.8: *6* rt.object_name_ref
def object_name_ref(self, module, obj, name=None, classes=None):
    """
    Link to the definition for 'module' using 'obj' with the optional 'name'
    used as the label (instead of the name of 'obj'). The optional 'classes'
    can be used to customise the CSS classes employed.
    """
    return self.name_link(
        module.full_name(),
        obj.full_name(),
        name or obj.name, classes)
#@+node:ekr.20150722204300.9: *6* rt.popup
def popup(self, classes, aList):

    self.span_list(classes or 'popup', aList)
#@+node:ekr.20150722204300.28: *6* rt.span
def span(self, class_name, wrap=False):

    if class_name in self.enable_list:
        self.newline()
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
            self.gen(f"<span class='%s'>" % (full_class_name))
        else:
            self.gen('<span>')
        # self.newline()
    self.div_stack.append(class_name)
#@+node:ekr.20150722224734.1: *6* rt.span_list & span_node
def span_list(self, class_name, aList, sep=None):

    self.span(class_name)
    self.visit_list(aList, sep=sep)
    self.end_span(class_name)

def span_node(self, class_name, node):

    self.span(class_name)
    self.visit(node)
    self.end_span(class_name)
#@+node:ekr.20150722204300.10: *6* rt.summary_link
def summary_link(self, module_name, full_name, name, classes=None):

    return self.name_link(
        f"{module_name}-summary", full_name, name, classes)
#@+node:ekr.20160315161259.1: *5* rt.main
def main(self, fn, node):
    """Return a report for the given ast node as a string."""
    self.gen(self.html_header % {
            'css-fn': self.css_fn,
            'title': f"Module: {fn}"
        })
    self.parent = None
    self.parents = [None]
    self.visit(node)
    self.gen(self.html_footer)
    return ''.join(self.code_list)
#@+node:ekr.20150722204300.44: *5* rt.visit
def visit(self, node):
    """Walk a tree of AST nodes."""
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    method(node)
#@+node:ekr.20150722204300.45: *5* rt.visit_list
def visit_list(self, aList, sep=None):
    # pylint: disable=arguments-differ
    if aList:
        for z in aList:
            self.visit(z)
            self.gen(sep)
        self.clean(sep)
#@+node:ekr.20150722204300.46: *5* rt.visitors
#@+node:ekr.20170721074613.1: *6* rt.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):

    self.div('statement')
    self.visit(node.target)
    self.op('=:', leading=True, trailing=True)
    self.visit(node.annotation)
    self.blank()
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.49: *6* rt.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):

    self.div('statement')
    self.keyword("assert")
    self.visit(node.test)
    if node.msg:
        self.comma()
        self.visit(node.msg)
    self.end_div('statement')
#@+node:ekr.20150722204300.50: *6* rt.Assign
def do_Assign(self, node):

    self.div('statement')
    for z in node.targets:
        self.visit(z)
        self.op('=', leading=True, trailing=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.51: *6* rt.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):

    self.visit(node.value)
    self.gen('.')
    self.gen(node.attr)
#@+node:ekr.20160523102939.1: *6* rt.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    self.div('statement')
    self.keyword('await')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.52: *6* rt.AugAssign
#  AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    op_name_ = op_name(node.op)
    self.div('statement')
    self.visit(node.target)
    self.op(op_name_, leading=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.53: *6* rt.BinOp
def do_BinOp(self, node):

    op_name_ = op_name(node.op)
    # self.span(op_name_)
    self.visit(node.left)
    self.op(op_name_, leading=True)
    self.visit(node.right)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.54: *6* rt.BoolOp
def do_BoolOp(self, node):

    op_name_ = op_name(node.op).strip()
    self.span(op_name_)
    for i, node2 in enumerate(node.values):
        if i > 0:
            self.keyword(op_name_)
        self.visit(node2)
    self.end_span(op_name_)
#@+node:ekr.20150722204300.55: *6* rt.Break
def do_Break(self, node):

    self.simple_statement('break')
#@+node:ekr.20160523103529.1: *6* rt.Bytes (Python 3)
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20150722204300.56: *6* rt.Call & do_keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    # self.span("callfunc")
    self.visit(node.func)
    # self.span("call")
    self.gen('(')
    self.visit_list(node.args, sep=',')
    if node.keywords:
        self.visit_list(node.keywords, sep=',')
    if getattr(node, 'starargs', None):
        self.op('*', trailing=False)
        self.visit(node.starargs)
        self.comma()
    if getattr(node, 'kwargs', None):
        self.op('**', trailing=False)
        self.visit(node.kwargs)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('call')
    # self.end_span('callfunc')
#@+node:ekr.20150722204300.57: *7* rt.do_keyword
# keyword = (identifier arg, expr value)
# keyword arguments supplied to call

def do_keyword(self, node):

    self.span('keyword-arg')
    self.gen(node.arg)
    self.blank()
    self.gen('=')
    self.blank()
    self.visit(node.value)
    self.end_span('keyword-arg')
#@+node:ekr.20150722204300.58: *6* rt.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node):

    has_bases = (node.bases or hasattr(node, 'keywords') or
        hasattr(node, 'starargs') or hasattr(node, 'kwargs'))
    self.div('class')
    self.keyword("class")
    self.gen(node.name)  # Always a string.
    if has_bases:
        self.gen('(')
        self.visit_list(node.bases, sep=', ')
        if getattr(node, 'keywords', None):  # Python 3
            for keyword in node.keywords:
                self.gen(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
        if getattr(node, 'starargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.starargs))
        if getattr(node, 'kwargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.kwargs))
        self.gen(')')
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('class')
#@+node:ekr.20150722204300.59: *6* rt.Compare
def do_Compare(self, node):

    assert len(node.ops) == len(node.comparators)
    # self.span('compare')
    self.visit(node.left)
    for i in range(len(node.ops)):
        op_name_ = op_name(node.ops[i])
        self.op(op_name_, leading=True)
        self.visit(node.comparators[i])
    # self.end_span('compare')
#@+node:ekr.20150722204300.60: *6* rt.comprehension
# comprehension = (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):

    self.visit(node.target)
    self.keyword('in')
    # self.span('collection')
    self.visit(node.iter)
    if node.ifs:
        self.keyword('if')
        # self.span_list("conditional", node.ifs, sep=' ')
        for z in node.ifs:
            self.visit(z)
            self.blank()
        self.clean(' ')
    # self.end_span('collection')
#@+node:ekr.20170721073431.1: *6* rt.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20150722204300.61: *6* rt.Continue
def do_Continue(self, node):

    self.simple_statement('continue')
#@+node:ekr.20150722204300.62: *6* rt.Delete
def do_Delete(self, node):

    self.div('statement')
    self.keyword('del')
    if node.targets:
        self.visit_list(node.targets, sep=',')
    self.end_div('statement')
#@+node:ekr.20150722204300.63: *6* rt.Dict
def do_Dict(self, node):

    assert len(node.keys) == len(node.values)
    # self.span('dict')
    self.gen('{')
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.colon()
        self.visit(node.values[i])
        self.comma()
    self.clean_comma()
    self.gen('}')
    # self.end_span('dict')
#@+node:ekr.20160523104330.1: *6* rt.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.47: *6* rt.do_arguments & helpers
# arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults)

def do_arguments(self, node):

    assert isinstance(node, ast.arguments), node
    first_default = len(node.args) - len(node.defaults)
    for n, arg in enumerate(node.args):
        if isinstance(arg, (list, tuple)):
            self.tuple_parameter(arg)
        else:
            self.visit(arg)
        if n >= first_default:
            default = node.defaults[n - first_default]
            self.gen("=")
            self.visit(default)
        self.comma()
    if getattr(node, 'vararg', None):
        self.gen('*')
        self.gen(self.name(node.vararg))
        self.comma()
    if getattr(node, 'kwarg', None):
        self.gen('**')
        self.gen(self.name(node.kwarg))
        self.comma()
    self.clean_comma()
#@+node:ekr.20160315182225.1: *7* rt.arg (Python 3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):

    self.gen(node.arg)
    if getattr(node, 'annotation', None):
        self.colon()
        self.visit(node.annotation)
#@+node:ekr.20150722204300.48: *7* rt.tuple_parameter
def tuple_parameter(self, node):

    assert isinstance(node, (list, tuple)), node
    self.gen("(")
    for param in node:
        if isinstance(param, tuple):
            self.tuple_parameter(param)
        else:
            self.visit(param)
    self.gen(")")
#@+node:ekr.20150722204300.64: *6* rt.Ellipsis
def do_Ellipsis(self, node):

    self.gen('...')
#@+node:ekr.20150722204300.65: *6* rt.ExceptHandler
def do_ExceptHandler(self, node):

    self.div('excepthandler')
    self.keyword("except")
    if not node.type:
        self.clean(' ')
    if node.type:
        self.visit(node.type)
    if node.name:
        self.keyword('as')
        self.visit(node.name)
    self.colon()
    self.div_body(node.body)
    self.end_div('excepthandler')
#@+node:ekr.20150722204300.66: *6* rt.Exec
# Python 2.x only.

def do_Exec(self, node):

    self.div('statement')
    self.keyword('exec')
    self.visit(node.body)
    if node.globals:
        self.comma()
        self.visit(node.globals)
    if node.locals:
        self.comma()
        self.visit(node.locals)
    self.end_div('statement')
#@+node:ekr.20150722204300.67: *6* rt.Expr
def do_Expr(self, node):

    self.div_node('expr', node.value)
#@+node:ekr.20160523103429.1: *6* rf.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s' % self.visit(node.body)
#@+node:ekr.20160523103751.1: *6* rt.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20150722204300.68: *6* rt.For & AsyncFor (Python 3)
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node, async_flag=False):

    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword("for")
    self.visit(node.target)
    self.keyword("in")
    self.visit(node.iter)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')

def do_AsyncFor(self, node):
    self.do_For(node, async_flag=True)
#@+node:ekr.20170721075845.1: *6* rf.FormattedValue (Python 3.6+: unfinished)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.div('statement')
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
    self.end_div('statement')
#@+node:ekr.20150722204300.69: *6* rt.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False):

    self.div('function', extra=f'id="%s"' % node.name)
    if async_flag:
        self.keyword('async')
    self.keyword("def")
    self.name(node.name)
    self.gen('(')
    self.visit(node.args)
    self.gen(')')
    if getattr(node, 'returns', None):
        self.blank()
        self.gen('->')
        self.blank()
        self.visit(node.returns)
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('function')

def do_AsyncFunctionDef(self, node):
    self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20150722204300.70: *6* rt.GeneratorExp
def do_GeneratorExp(self, node):

    # self.span('genexpr')
    self.gen('(')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span_node('item', node.elt)
    self.visit(node.elt)
    # self.span_list('generators', node.generators)
    self.visit_list(node.generators)
    self.gen(')')
    # self.end_span('genexpr')
#@+node:ekr.20150722204300.71: *6* rt.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert isinstance(ast2, ast.alias), repr(ast2)
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20150722204300.72: *6* rt.Global
def do_Global(self, node):

    self.div('statement')
    self.keyword("global")
    for z in node.names:
        self.gen(z)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20150722204300.73: *6* rt.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node, elif_flag=False):
    
    self.div('statement')
    self.keyword('elif' if elif_flag else 'if')
    self.visit(node.test)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        node1 = node.orelse[0]
        if isinstance(node1, ast.If) and len(node.orelse) == 1:
            self.do_If(node1, elif_flag=True)
        else:
            self.keyword('else')
            self.colon()
            self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.74: *6* rt.IfExp (TernaryOp)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):

    # self.span('ifexp')
    self.visit(node.body)
    self.keyword('if')
    self.visit(node.test)
    self.keyword('else')
    self.visit(node.orelse)
    # self.end_span('ifexp')
#@+node:ekr.20150722204300.75: *6* rt.Import
def do_Import(self, node):

    self.div('statement')
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)  # self.gen(self.module_link(name))
        if alias:
            self.keyword("as")
            self.name(alias)
    self.end_div('statement')
#@+node:ekr.20150722204300.76: *6* rt.ImportFrom
def do_ImportFrom(self, node):

    self.div('statement')
    self.keyword("from")
    self.gen(self.module_link(node.module))
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)
        if alias:
            self.keyword("as")
            self.name(alias)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20160315190818.1: *6* rt.Index
def do_Index(self, node):

    self.visit(node.value)
#@+node:ekr.20170721080959.1: *6* rf.JoinedStr (Python 3.6+: unfinished)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20150722204300.77: *6* rt.Lambda
def do_Lambda(self, node):

    # self.span('lambda')
    self.keyword('lambda')
    self.visit(node.args)
    self.comma()
    self.span_node("code", node.body)
    # self.end_span('lambda')
#@+node:ekr.20150722204300.78: *6* rt.List
# List(expr* elts, expr_context ctx)

def do_List(self, node):

    # self.span('list')
    self.gen('[')
    if node.elts:
        for z in node.elts:
            self.visit(z)
            self.comma()
        self.clean_comma()
    self.gen(']')
    # self.end_span('list')
#@+node:ekr.20150722204300.79: *6* rt.ListComp
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):

    # self.span('listcomp')
    self.gen('[')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span('ifgenerators')
    self.visit_list(node.generators)
    self.gen(']')
    # self.end_span('ifgenerators')
    # self.end_span('listcomp')
#@+node:ekr.20150722204300.80: *6* rt.Module
def do_Module(self, node):

    self.doc(node)
    self.visit_list(node.body)
#@+node:ekr.20150722204300.81: *6* rt.Name
def do_Name(self, node):

    self.name(node.id)
#@+node:ekr.20160315165109.1: *6* rt.NameConstant
def do_NameConstant(self, node):  # Python 3 only.

    self.name(repr(node.value))
#@+node:ekr.20160317051849.2: *6* rt.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    self.div('statement')
    self.keyword('nonlocal')
    self.gen(', '.join(node.names))
    self.end_div('statement')
#@+node:ekr.20150722204300.82: *6* rt.Num
def do_Num(self, node):

    self.gen(self.text(repr(node.n)))
#@+node:ekr.20150722204300.83: *6* rt.Pass
def do_Pass(self, node):

    self.simple_statement('pass')
#@+node:ekr.20150722204300.84: *6* rt.Print
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):

    self.div('statement')
    self.keyword("print")
    self.gen('(')
    if node.dest:
        self.op('>>\n')
        self.visit(node.dest)
        self.comma()
        self.newline()
        if node.values:
            for z in node.values:
                self.visit(z)
                self.comma()
                self.newline()
    self.clean('\n')
    self.clean_comma()
    self.gen(')')
    self.end_div('statement')
#@+node:ekr.20150722204300.85: *6* rt.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    self.div('statement')
    self.keyword("raise")
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            self.visit(getattr(node, attr))
    self.end_div('statement')
#@+node:ekr.20160523105022.1: *6* rt.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20150722204300.86: *6* rt.Return
def do_Return(self, node):

    self.div('statement')
    self.keyword("return")
    if node.value:
        self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160523104433.1: *6* rt.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523104454.1: *6* rt.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.87: *6* rt.Slice
def do_Slice(self, node):

    # self.span("slice")
    if node.lower:
        self.visit(node.lower)
    self.colon()
    if node.upper:
        self.visit(node.upper)
    if node.step:
        self.colon()
        self.visit(node.step)
    # self.end_span("slice")
#@+node:ekr.20160317051849.3: *6* rt.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.gen('*')
    self.visit(node.value)
#@+node:ekr.20150722204300.88: *6* rt.Str
def do_Str(self, node):
    """This represents a string constant."""

    def clean(s):
        return s.replace(' ', '').replace('\n', '').replace('"', '').replace("'", '')

    assert isinstance(node.s, str)
    if self.last_doc and clean(self.last_doc) == clean(node.s):
        # Already seen.
        self.last_doc = None
    else:
        self.string(node.s)
#@+node:ekr.20150722204300.89: *6* rt.Subscript
def do_Subscript(self, node):

    # self.span("subscript")
    self.visit(node.value)
    self.gen('[')
    self.visit(node.slice)
    self.gen(']')
    # self.end_span("subscript")
#@+node:ekr.20160315190913.1: *6* rt.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    if node.finalbody:
        self.keyword('finally')
        self.colon()
        self.div_list('body', node.finalbody)
    self.end_div('statement')
#@+node:ekr.20150722204300.90: *6* rt.TryExcept
def do_TryExcept(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.div_body(node.handlers)
    self.end_div('statement')
#@+node:ekr.20150722204300.91: *6* rt.TryFinally
def do_TryFinally(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_body(node.body)
    self.keyword('finally')
    self.colon()
    self.div_body(node.final.body)
    self.end_div('statement')
#@+node:ekr.20150722204300.92: *6* rt.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):

    # self.span('tuple')
    self.gen('(')
    for z in node.elts or []:
        self.visit(z)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('tuple')
#@+node:ekr.20150722204300.94: *6* rt.While
def do_While(self, node):

    self.div('statement')
    self.div(None)
    self.keyword("while")
    self.visit(node.test)
    self.colon()
    self.end_div(None)
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.93: *6* rt.UnaryOp
def do_UnaryOp(self, node):

    op_name_ = op_name(node.op).strip()
    # self.span(op_name_)
    self.op(op_name_, trailing=False)
    self.visit(node.operand)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.95: *6* rt.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):

    context_expr = getattr(node, 'context_expr', None)
    optional_vars = getattr(node, 'optional_vars', None)
    items = getattr(node, 'items', None)
    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword('with')
    if context_expr:
        self.visit(context_expr)
    if optional_vars:
        self.keyword('as')
        self.visit_list(optional_vars)
    if items:
        for item in items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                self.keyword('as')
                self.visit(item.optional_vars)
    self.colon()
    self.div_body(node.body)
    self.end_div('statement')

def do_AsyncWith(self, node):
    self.do_With(node, async_flag=True)
#@+node:ekr.20150722204300.96: *6* rt.Yield
def do_Yield(self, node):

    self.div('statement')
    self.keyword('yield')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160317051849.5: *6* rt.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    self.div('statement')
    self.keyword('yield from')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160225102931.1: *3* class TokenSync (deprecated)
class TokenSync:
    """A class to sync and remember tokens."""
    # To do: handle comments, line breaks...
    @others
#@+node:ekr.20160225102931.2: *4*  ts.ctor & helpers
def __init__(self, s, tokens):
    """Ctor for TokenSync class."""
    assert isinstance(tokens, list)  # Not a generator.
    self.s = s
    self.first_leading_line = None
    self.lines = [z.rstrip() for z in g.splitLines(s)]
    # Order is important from here on...
    self.nl_token = self.make_nl_token()
    self.line_tokens = self.make_line_tokens(tokens)
    self.blank_lines = self.make_blank_lines()
    self.string_tokens = self.make_string_tokens()
    self.ignored_lines = self.make_ignored_lines()
#@+node:ekr.20160225102931.3: *5* ts.make_blank_lines
def make_blank_lines(self):
    """Return of list of line numbers of blank lines."""
    result = []
    for i, aList in enumerate(self.line_tokens):
        # if any([self.token_kind(z) == 'nl' for z in aList]):
        if len(aList) == 1 and self.token_kind(aList[0]) == 'nl':
            result.append(i)
    return result
#@+node:ekr.20160225102931.4: *5* ts.make_ignored_lines
def make_ignored_lines(self):
    """
    Return a copy of line_tokens containing ignored lines,
    that is, full-line comments or blank lines.
    These are the lines returned by leading_lines().
    """
    result = []
    for i, aList in enumerate(self.line_tokens):
        for z in aList:
            if self.is_line_comment(z):
                result.append(z)
                break
        else:
            if i in self.blank_lines:
                result.append(self.nl_token)
            else:
                result.append(None)
    assert len(result) == len(self.line_tokens)
    for i, aList in enumerate(result):
        if aList:
            self.first_leading_line = i
            break
    else:
        self.first_leading_line = len(result)
    return result
#@+node:ekr.20160225102931.5: *5* ts.make_line_tokens (trace tokens)
def make_line_tokens(self, tokens):
    """
    Return a list of lists of tokens for each list in self.lines.
    The strings in self.lines may end in a backslash, so care is needed.
    """
    import token as tm
    n, result = len(self.lines), []
    for i in range(0, n+1):
        result.append([])
    for token in tokens:
        t1, t2, t3, t4, t5 = token
        kind = tm.tok_name[t1].lower()
        srow, scol = t3
        erow, ecol = t4
        line = erow - 1 if kind == 'string' else srow - 1
        result[line].append(token)
    assert len(self.lines) + 1 == len(result), len(result)
    return result
#@+node:ekr.20160225102931.6: *5* ts.make_nl_token
def make_nl_token(self):
    """Return a newline token with '\n' as both val and raw_val."""
    import token as tm
    t1 = tm.NEWLINE
    t2 = '\n'
    t3 = (0, 0)  # Not used.
    t4 = (0, 0)  # Not used.
    t5 = '\n'
    return t1, t2, t3, t4, t5
#@+node:ekr.20160225102931.7: *5* ts.make_string_tokens
def make_string_tokens(self):
    """Return a copy of line_tokens containing only string tokens."""
    result = []
    for aList in self.line_tokens:
        result.append([z for z in aList if self.token_kind(z) == 'string'])
    assert len(result) == len(self.line_tokens)
    return result
#@+node:ekr.20160225102931.8: *4* ts.check_strings
def check_strings(self):
    """Check that all strings have been consumed."""
    for i, aList in enumerate(self.string_tokens):
        if aList:
            g.trace(f"warning: line {i}. unused strings: {aList}")
#@+node:ekr.20160225102931.10: *4* ts.is_line_comment
def is_line_comment(self, token):
    """Return True if the token represents a full-line comment."""
    import token as tm
    t1, t2, t3, t4, t5 = token
    kind = tm.tok_name[t1].lower()
    raw_val = t5
    return kind == 'comment' and raw_val.lstrip().startswith('#')
#@+node:ekr.20160225102931.12: *4* ts.last_node
def last_node(self, node):
    """Return the node of node's tree with the largest lineno field."""

    class LineWalker(ast.NodeVisitor):

        def __init__(self):
            """Ctor for LineWalker class."""
            self.node = None
            self.lineno = -1

        def visit(self, node):
            """LineWalker.visit."""
            if hasattr(node, 'lineno'):
                if node.lineno > self.lineno:
                    self.lineno = node.lineno
                    self.node = node
            if isinstance(node, list):
                for z in node:
                    self.visit(z)
            else:
                self.generic_visit(node)

    w = LineWalker()
    w.visit(node)
    return w.node
#@+node:ekr.20160225102931.13: *4* ts.leading_lines
def leading_lines(self, node):
    """Return a list of the preceding comment and blank lines"""
    # This can be called on arbitrary nodes.
    leading = []
    if hasattr(node, 'lineno'):
        i, n = self.first_leading_line, node.lineno
        while i < n:
            token = self.ignored_lines[i]
            if token:
                s = self.token_raw_val(token).rstrip() + '\n'
                leading.append(s)
            i += 1
        self.first_leading_line = i
    return leading
#@+node:ekr.20160225102931.14: *4* ts.leading_string
def leading_string(self, node):
    """Return a string containing all lines preceding node."""
    return ''.join(self.leading_lines(node))
#@+node:ekr.20160225102931.15: *4* ts.line_at
def line_at(self, node, continued_lines=True):
    """Return the lines at the node, possibly including continuation lines."""
    n = getattr(node, 'lineno', None)
    if n is None:
        return f'<no line> for %s' % node.__class__.__name__
    if continued_lines:
        aList, n = [], n - 1
        while n < len(self.lines):
            s = self.lines[n]
            if s.endswith('\\'):
                aList.append(s[:-1])
                n += 1
            else:
                aList.append(s)
                break
        return ''.join(aList)
    return self.lines[n - 1]
#@+node:ekr.20160225102931.16: *4* ts.sync_string
def sync_string(self, node):
    """Return the spelling of the string at the given node."""
    n = node.lineno
    tokens = self.string_tokens[n - 1]
    if tokens:
        token = tokens.pop(0)
        self.string_tokens[n - 1] = tokens
        return self.token_val(token)
    g.trace('===== underflow', n, node.s)
    return node.s
#@+node:ekr.20160225102931.18: *4* ts.tokens_for_statement
def tokens_for_statement(self, node):
    assert isinstance(node, ast.AST), node
    name = node.__class__.__name__
    if hasattr(node, 'lineno'):
        tokens = self.line_tokens[node.lineno - 1]
        g.trace(' '.join([self.dump_token(z) for z in tokens]))
    else:
        g.trace('no lineno', name)
#@+node:ekr.20160225102931.19: *4* ts.trailing_comment
def trailing_comment(self, node):
    """
    Return a string containing the trailing comment for the node, if any.
    The string always ends with a newline.
    """
    if hasattr(node, 'lineno'):
        return self.trailing_comment_at_lineno(node.lineno)
    g.trace('no lineno', node.__class__.__name__, g.callers())
    return '\n'
#@+node:ekr.20160225102931.20: *4* ts.trailing_comment_at_lineno
def trailing_comment_at_lineno(self, lineno):
    """Return any trailing comment at the given node.lineno."""
    tokens = self.line_tokens[lineno - 1]
    for token in tokens:
        if self.token_kind(token) == 'comment':
            raw_val = self.token_raw_val(token).rstrip()
            if not raw_val.strip().startswith('#'):
                val = self.token_val(token).rstrip()
                s = f' %s\n' % val
                return s
    return '\n'
#@+node:ekr.20160225102931.21: *4* ts.trailing_lines
def trailing_lines(self):
    """return any remaining ignored lines."""
    trailing = []
    i = self.first_leading_line
    while i < len(self.ignored_lines):
        token = self.ignored_lines[i]
        if token:
            s = self.token_raw_val(token).rstrip() + '\n'
            trailing.append(s)
        i += 1
    self.first_leading_line = i
    return trailing
#@+node:ekr.20191122105543.1: *4* ts:dumps
#@+node:ekr.20160225102931.9: *5* ts.dump_token
def dump_token(self, token, verbose=False):
    """Dump the token. It is either a string or a 5-tuple."""
    import token as tm
    if isinstance(token, str):
        return token
    t1, t2, t3, t4, t5 = token
    kind = g.toUnicode(tm.tok_name[t1].lower())
    # raw_val = g.toUnicode(t5)
    val = g.toUnicode(t2)
    if verbose:
        return f'token: %10s %r' % (kind, val)
    return val
#@+node:ekr.20160225102931.17: *5* ts.token_kind/raw_val/val
def token_kind(self, token):
    """Return the token's type."""
    t1, t2, t3, t4, t5 = token
    import token as tm
    return g.toUnicode(tm.tok_name[t1].lower())

def token_raw_val(self, token):
    """Return the value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t5)

def token_val(self, token):
    """Return the raw value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t2)
#@+node:ekr.20160225102931.11: *5* ts.join
def join(self, aList, sep=','):
    """return the items of the list joined by sep string."""
    tokens = []
    for i, token in enumerate(aList or []):
        tokens.append(token)
        if i < len(aList) - 1:
            tokens.append(sep)
    return tokens
#@+node:ekr.20190910081550.1: *3* class SyntaxSanitizer
class SyntaxSanitizer:

    << SyntaxSanitizer docstring >>

    def __init__(self, c, keep_comments):
        self.c = c
        self.keep_comments = keep_comments

    @others
#@+node:ekr.20190910093739.1: *4* << SyntaxSanitizer docstring >>
r"""
This class converts section references, @others and Leo directives to
comments. This allows ast.parse to handle the result.

Within section references, these comments must *usually* be executable:
    
BEFORE:
    if condition:
        <\< do something >\>
AFTER:
    if condition:
        pass # do something
        
Alas, sanitation can result in a syntax error. For example, leoTips.py contains:
    
BEFORE:
    tips = [
        <\< define tips >\>
        ]

AFTER:
    tips = [
        pass # define tips
    ]
    
This fails because tips = [pass] is a SyntaxError.

The beautify* and black* commands clearly report such failures.
"""
#@+node:ekr.20190910022637.2: *4* sanitize.comment_leo_lines
def comment_leo_lines(self, p=None, s0=None):
    """
    Replace lines containing Leonine syntax with **special comment lines** of the form:
        
        {lws}#{lws}{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    This convention allows uncomment_special_lines to restore these lines.
    """
    # Choose a marker that appears nowhere in s.
    if p:
        s0 = p.b
    n = 5
    while('#'+ ('!'*n)) in s0:
        n += 1
    comment = '#' + ('!' * n)
    # Create a dict of directives.
    d = {z: True for z in g.globalDirectiveList}
    # Convert all Leonine lines to special comments.
    i, lines, result = 0, g.splitLines(s0), []
    while i < len(lines):
        progress = i
        s = lines[i]
        s_lstrip = s.lstrip()
        # Comment out any containing a section reference.
        j = s.find('<<')
        k = s.find('>>') if j > -1 else -1
        if -1 < j < k:
            result.append(comment+s)
            # Generate a properly-indented pass line.
            j2 = g.skip_ws(s, 0)
            result.append(f'{" "*j2}pass\n')
        elif s_lstrip.startswith('@'):
            # Comment out all other Leonine constructs.
            if self.starts_doc_part(s):
                # Comment the entire doc part, until @c or @code.
                result.append(comment+s)
                i += 1
                while i < len(lines):
                    s = lines[i]
                    result.append(comment+s)
                    i += 1
                    if self.ends_doc_part(s):
                        break
            else:
                j = g.skip_ws(s, 0)
                assert s[j] == '@'
                j += 1
                k = g.skip_id(s, j, chars='-')
                if k > j:
                    word = s[j : k]
                    if word == 'others':
                        # Remember the original @others line.
                        result.append(comment+s)
                        # Generate a properly-indented pass line.
                        result.append(f'{" "*(j-1)}pass\n')
                    else:
                        # Comment only Leo directives, not decorators.
                        result.append(comment+s if word in d else s)
                else:
                    result.append(s)
        elif s_lstrip.startswith('#') and self.keep_comments:
            # A leading comment.
            # Bug fix: Preserve lws in comments, too.
            j2 = g.skip_ws(s, 0)
            result.append(" "*j2+comment+s)
        else:
            # A plain line.
            result.append(s)
        if i == progress:
            i += 1
    return comment, ''.join(result)
#@+node:ekr.20190910022637.3: *4* sanitize.starts_doc_part & ends_doc_part
def starts_doc_part(self, s):
    """Return True if s word matches @ or @doc."""
    return s.startswith(('@\n', '@doc\n', '@ ', '@doc '))

def ends_doc_part(self, s):
    """Return True if s word matches @c or @code."""
    return s.startswith(('@c\n', '@code\n', '@c ', '@code '))
#@+node:ekr.20190910022637.4: *4* sanitize.uncomment_leo_lines
def uncomment_leo_lines(self, comment, p, s0):
    """Reverse the effect of comment_leo_lines."""
    lines = g.splitLines(s0)
    i, result = 0, []
    while i < len(lines):
        progress = i
        s = lines[i]
        i += 1
        if comment in s:
            # One or more special lines.
            i = self.uncomment_special_lines(comment, i, lines, p, result, s)
        else:
            # A regular line.
            result.append(s)
        assert progress < i
    return ''.join(result).rstrip() + '\n'
#@+node:ekr.20190910022637.5: *4* sanitize.uncomment_special_line & helpers
def uncomment_special_lines(self, comment, i, lines, p, result, s):
    """
    This method restores original lines from the special comment lines
    created by comment_leo_lines. These lines have the form:
        
        {lws}#{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    s is a line containing the comment delim.
    i points at the *next* line.
    Handle one or more lines, appending stripped lines to result.
    """
    #
    # Delete the lws before the comment.
    # This works because the tail contains the original whitespace.
    assert comment in s
    s = s.lstrip().replace(comment, '')
    #
    # Here, s is the original line.
    if comment in s:
        g.trace(f"can not happen: {s!r}")
        return i
    if self.starts_doc_part(s):
        result.append(s)
        while i < len(lines):
            s = lines[i].lstrip().replace(comment, '')
            i += 1
            result.append(s)
            if self.ends_doc_part(s):
                break
        return i
    j = s.find('<<')
    k = s.find('>>') if j > -1 else -1
    if -1 < j < k or '@others' in s:
        #
        # A section reference line or an @others line.
        # Such lines are followed by a pass line.
        #
        # The beautifier may insert blank lines before the pass line.
        kind = 'section ref' if -1 < j < k else '@others'
        # Restore the original line, including leading whitespace.
        result.append(s)
        # Skip blank lines.
        while i < len(lines) and not lines[i].strip():
            i += 1
        # Skip the pass line.
        if i < len(lines) and lines[i].lstrip().startswith('pass'):
            i += 1
        else:
            g.trace(f"*** no pass after {kind}: {p.h}")
    else:
        # A directive line or a comment line.
        result.append(s)
    return i
#@+node:ekr.20200729081336.1: *3* re TOG classes
#@+node:ekr.20191113133338.1: *4* class TestRunner
class TestRunner:
    """
    A testing framework for TokenOrderGenerator and related classes.
    """
    
    counts, times = {}, {}
    << define valid actions & flags >>
    @others
   
#@+node:ekr.20191222064729.1: *5* << define valid actions & flags >>
valid_actions = [
    'run-ast-tokens',       # Alternate pass 0.
    'make-tokens-and-tree', # Pass 0.
    'create-links',         # Pass 1.
    'fstringify',           # Pass 2.
    # Dumps...
    'dump-all',
    'dump-ast', # Was dump-raw-tree.
    'dump-contents',
    'dump-lines',
    'dump-results',
    'dump-times',
    'dump-tokens',
    'dump-tree',
]

valid_flags = [
    'all',
    'all-leo-files',
    'coverage',
    'dump-all-after-fail',
    'dump-ast-tree-first',
    'dump-results',
    'dump-tokens-after-fail',
    'dump-tokens-first',
    'dump-tree-after-fail',
    'no-trace-after-fail',
    'set-trace-mode',
    'show-pass0-times',
    'show-create-links-time',
    'show-fstringify-time',
    'show-exception-after-fail',
    'show-make-tokens-time',
    'show-test-description',
    'show-test-kind',
    'summarize',
    'trace-tokenizer-tokens',
    'verbose-fail',
]
#@+node:ekr.20191205160754.4: *5* TR.run_tests & helpers
def run_tests(self, actions, flags, root, contents=None):
    """The outer test runner."""
    # Startup.
    self.fails = []
    self.root = root
    self.times = {}
    # Create self.actions and self.flags.
    ok = self.make_actions_and_flags(actions, flags)
    if not ok:
        print('Aborting...')
        return
    flags = self.flags
    self.show_status()
    if contents:
        self.tests = [(contents, root.h or 'None')]
    elif 'all-leo-files' in flags:
        self.tests = self.make_leo_tests()
    else:
        self.tests = self.make_tests(root)
    # Execute all tests.
    t1 = get_time()
    for contents, description in self.tests:
        # run_one_test catches all exceptions.
        if 'show-test-description' in flags:
            print(f"Running {description}...")
        ok = self.run_one_test(contents, description)
        if not ok:
            self.fails.append(description)
        if 'fail-fast' in flags:
            break
    # End-of-tests reports.
    t2 = get_time()
    self.times['total_time'] = t2 - t1
    if 'coverage' in flags:
        self.show_coverage()
    if 'summarize' in flags:
        self.summarize()
#@+node:ekr.20191205163727.1: *6* TR.make_actions_and_flags
def make_actions_and_flags(self, actions, flags):
    """
    Create self.actions and self.flags.
    
    Return False if there are unknow actions or flags.
    """
    valid_actions, valid_flags = self.valid_actions, self.valid_flags
    # Check valid actions.
    for z in valid_actions:
        assert hasattr(self, z.replace('-','_')), repr(z)
    # Clean and check actions.
    self.actions = [z for z in actions if z in valid_actions]
    bad_actions = [z for z in actions if z not in valid_actions]
    if bad_actions:
        for z in bad_actions:
            print('Unknown action:', z)
        return False
    # Clean and check flags.
    flags = [z.lower() for z in flags or []]
    self.flags = [z for z in flags if z in valid_flags]
    bad_flags = [z for z in flags if z not in valid_flags]
    if bad_flags:
        for z in bad_flags:
            print('Unknown flag:', z)
        return False
    return True
#@+node:ekr.20191205172431.1: *6* TR.make_leo_tests
def make_leo_tests(self):
    """
    Leo-specific code for unit tests.
    
    Return a list of tuples (contents, description) for all of Leo's core
    .py files.
    """
    import leo.core.leoGlobals as leo_g
    core_directory = leo_g.os_path_finalize_join(leo_g.app.loadDir, '..', 'core')
    assert os.path.exists(core_directory), core_directory
    paths = glob.glob(core_directory + os.path.sep + 'leo*.py')
    tests = []
    for path in paths:
        assert os.path.exists(path), path
        with open(path, 'r') as f:
            contents = f.read()
        description = path
        tests.append((contents, description))   
    return tests

#@+node:ekr.20191205160754.2: *6* TR.make_tests
def make_tests(self, root):
    """
    Leo-specific code for unit tests.
    
    Return a list of tuples (contents, description) found in all children
    of the root, except this node.
    """
    import leo.core.leoGlobals as leo_g
    tests = []
    contents_tag = 'test:'
    file_tag = 'file:'
    after = root.nodeAfterTree()
    p = root.copy()
    while p and p != after:
        if p.h.startswith(('fail:', 'fails')):
            # Ignore all fails, regardless of 'all' flag.
            p.moveToNodeAfterTree()
        elif 'all' not in self.flags and p.h.startswith('ignore:'):
            # Honor 'ignore' only when *not* runnining all tests.
            p.moveToNodeAfterTree()
        elif p.h.startswith(contents_tag):
            description = p.h
            contents = p.b.strip() + '\n'
            tests.append((contents, description))
            p.moveToThreadNext()
        elif p.h.startswith(file_tag):
            description = p.h
            s = p.h[len(file_tag):].strip()
            parts = [leo_g.app.loadDir, '..'] + s.split('..')
            path = os.path.sep.join(parts)
            if os.path.exists(path):
                with open(path, 'r') as f:
                    contents = f.read()
                tests.append((contents, description))
                p.moveToThreadNext()
            else:
                assert False, f"file not found: {path}"
        else:
            # Ignore organizer nodes.
            p.moveToThreadNext()
    if not tests:
        print(f"no tests in {root.h}")
    return tests
#@+node:ekr.20191122025155.1: *6* TR.show_coverage
def show_coverage(self):
    if self.toi:
        self.toi.report_coverage()
#@+node:ekr.20191205160754.5: *6* TR.show_status
def show_status(self):
    """Show the preliminary status."""
    flags = self.flags
    print('')
    if 'show-test-kind' in flags:
        if 'all-leo-files' in flags:
            kind = 'Testing all Leo files'
        elif 'all' in flags:
            kind = 'Running *all* unit tests'
        else:
            kind = 'Running *selected* unit tests'
        print(f"{self.root.h}: {kind}...")
    if 'run-ast-tokens' in self.actions:
        print('\nUsing asttokens, *not* the TOG classes')
#@+node:ekr.20191205160754.6: *6* TR.summarize
def summarize(self):
    fails, tests = self.fails, self.tests
    status = 'FAIL' if fails else 'PASS'
    if fails:
        print('')
        g.printObj(fails, tag='Failed tests')
    print(
        f"\n{status} Ran "
        f"{len(tests)} test{g.plural(len(tests))}")
    if not 'dump-times' in self.flags:
        self.dump_times()
#@+node:ekr.20191122021515.1: *5* TR.run_one_test
def run_one_test(self, contents, description):
    """
    Run the test given by the contents and description.
    """
    tag = 'run_tests'
    self.description = description
    # flags = self.flags
    # Clean the contents.
    self.contents = contents = contents.strip() + '\n'
    
    #
    # Execute actions, in the user-defined order.
    bad_actions = []
    for action in self.actions:
        helper = getattr(self, action.replace('-', '_'), None)
        if helper:
            try:
                helper()
            except Exception as e:
                print(f"{tag}: Exception in {action}: {e}")
                if 'show-exception-after-fail' in self.flags:
                    g.es_exception()
                return False
        else:
            bad_actions.append(action)
    if bad_actions:
        for action in list(set(bad_actions)):
            print(f"{tag}: bad action option: {action!r}")
    return True
#@+node:ekr.20191205160624.1: *5* TR: actions...
# Actions should fail by throwing an exception.
#@+node:ekr.20191226064933.1: *6* TR.create_links (pass 1)
def create_links(self):
    """Pass 1: TOG.create_links"""
    flags, toi = self.flags, self.toi
    # Catch exceptions so we can get data late.
    try:
        t1 = get_time()
        # Yes, list *is* required here.
        list(toi.create_links(self.tokens, self.tree, file_name=self.description))
        t2 = get_time()
        self.update_times('10: create-links', t2 - t1)
    except Exception as e:
        g.trace(f"\nFAIL: make-tokens\n")
        # Don't use g.trace.  It doesn't handle newlines properly.
        print(e)
        if 'show-exception-after-fail' in flags:
            g.es_exception()
        if 'dump-all-after-fail' in flags:
            self.dump_all()
        else:
            if 'dump-tokens-after-fail' in flags:
                self.dump_tokens()
            if 'dump-tree-after-fail' in flags:
                self.dump_tree()
        if 'no-trace-after-fail':
            toi.trace_mode = False
        raise
#@+node:ekr.20191122022728.1: *6* TR.dump_all
def dump_all(self):

    if self.toi:
        self.dump_contents()
        self.dump_tokens()
        self.dump_tree()
        # self.dump_ast()

#@+node:ekr.20191122025306.2: *6* TR.dump_ast
def dump_ast(self):
    """Dump an ast tree.  Similar to ast.dump()."""
    print('\nast tree...\n')
    print(AstDumper().dump_ast(self.tree))
    print('')
#@+node:ekr.20191122025303.1: *6* TR.dump_contents
def dump_contents(self):
    contents = self.contents
    print('\nContents...\n')
    for i, z in enumerate(g.splitLines(contents)):
        print(f"{i+1:<3} ", z.rstrip())
    print('')
#@+node:ekr.20191122025306.1: *6* TR.dump_lines
def dump_lines(self):
    print('\nTOKEN lines...\n')
    for z in self.tokens:
        if z.line.strip():
            print(z.line.rstrip())
        else:
            print(repr(z.line))
    print('')
#@+node:ekr.20191225063758.1: *6* TR.dump_results
def dump_results(self):
    print('\nResults...\n')
    print(tokens_to_string(self.tokens))
#@+node:ekr.20191226095129.1: *6* TR.dump_times
def dump_times(self):
    """
    Show all calculated times.
    
    Keys should start with a priority (sort order) of the form `[0-9][0-9]:`
    """
    if not self.times:
        return
    print('')
    for key in sorted(self.times):
        t = self.times.get(key)
        key2 = key[3:]
        print(f"{key2:>16}: {t:6.3f} sec.")
#@+node:ekr.20191122025418.1: *6* TR.dump_tokens
def dump_tokens(self):
    tokens = self.tokens
    print('\nTokens...\n')
    print("Note: values shown are repr(value) *except* for 'string' tokens.\n")
    # pylint: disable=not-an-iterable
    if self.toi:
        for z in tokens:
            print(z.dump())
        print('')
    else:
        import token as tm
        for z in tokens:
            kind = tm.tok_name[z.type].lower()
            print(f"{z.index:4} {kind:>12} {z.string!r}")
#@+node:ekr.20191122025419.1: *6* TR.dump_tree
def dump_tree(self):
    print('\nPatched tree...\n')
    tokens, tree = self.tokens, self.tree
    if self.toi:
        print(dump_tree_and_links(tree))
        return
    try:
        # pylint: disable=import-error
        from asttokens.util import walk
    except Exception:
        return
    for z in walk(tree):
        class_name = z.__class__.__name__
        first, last = z.first_token.index, z.last_token.index
        token_range = f"{first:>4}..{last:<4}"
        if isinstance(z, ast.Module):
            tokens_s = ''
        else:
            tokens_s = ' '.join(
                repr(z.string) for z in tokens[first:last] if z)
        print(f"{class_name:>12} {token_range:<10} {tokens_s}")
#@+node:ekr.20191222074711.1: *6* TR.fstringify (pass 2)
def fstringify(self):
    """Pass 2: TOG.fstringify."""
    toi = self.toi
    assert isinstance(toi, TokenOrderGenerator), repr(toi)
    t1 = get_time()
    toi.fstringify(toi.tokens, toi.tree, filename='unit test')
    t2 = get_time()
    self.update_times('20: fstringify', t2 - t1)
#@+node:ekr.20191226063007.1: *6* TR.make_tokens_and_tree (pass 0)
def make_tokens_and_tree(self):
    """Pass 0: TOG.make_tokens_and_tree."""
    contents, flags = self.contents, self.flags
    t1 = get_time()
    # Create and remember the toi.
    toi = self.toi = TokenOrderInjector()
    toi.trace_mode = 'set-trace-mode' in flags
    # Tokenize.
    self.tokens = make_tokens(contents)
    t2 = get_time()
    self.update_times('01: make-tokens', t2 - t1)
    # Parse.
    self.tree = parse_ast(contents)
    t3 = get_time()
    self.update_times('01: parse-ast', t3 - t2)
    # Dump.
    if 'dump-tokens-first' in flags:
        dump_tokens(self.tokens)
    if 'dump-ast-tree-first' in flags:
        dump_ast(self.tree)
#@+node:ekr.20191226063942.1: *6* TR.run_ast_tokens
def run_ast_tokens(self):
    # pylint: disable=import-error
    # It's ok to raise ImportError here.
    import asttokens
    t1 = get_time()
    atok = asttokens.ASTTokens(self.contents, parse=True)
    self.tree = atok.tree
    self.tokens = atok._tokens
    t2 = get_time()
    self.update_times('01: ast-tokens', t2 - t1)
#@+node:ekr.20191228183156.1: *6* TR.update_counts & update_times
def update_counts(self, key, n):
    """Update the count statistic given by key, n."""
    old_n = self.times.get(key, 0)
    self.counts [key] = old_n + n

def update_times(self, key, t):
    """Update the timing statistic given by key, t."""
    old_t = self.times.get(key, 0.0)
    self.times [key] = old_t + t
#@+node:ekr.20191113054314.1: *4* class TokenOrderInjector (TOG)
class TokenOrderInjector (TokenOrderGenerator):
    """
    A class that injects parent/child data into tokens and ast nodes.
    """
    @others
#@+node:ekr.20191113054550.1: *5* toi.begin_visitor
def begin_visitor(self, node):
    """
    TokenOrderInjector.begin_visitor.
    
    Enter a visitor, inject data into the ast node, and update stats.
    """
    #
    # Do this first, *before* updating self.node.
    self.coverage_set.add(node.__class__.__name__)
    node.parent = self.node
    if self.node:
        children = getattr(self.node, 'children', [])
        children.append(node)
        self.node.children = children
    #
    # *Now* update self.node, etc.
    super().begin_visitor(node)
#@+node:ekr.20191121122230.1: *4* class TokenOrderNodeGenerator (TOG)
class TokenOrderNodeGenerator(TokenOrderGenerator):
    """A class that yields a stream of nodes."""

    # Other overrides...
    def sync_token(self, kind, val):
        pass
        
    @others
#@+node:ekr.20191228153344.1: *5* tong.generate_nodes
def generate_nodes(self, tokens, tree, file_name=''):
    """Entry: yield a stream of nodes."""
    #
    # Init all ivars.
    self.file_name = file_name
        # For tests.
    self.level = 0
        # Python indentation level.
    self.node = None
        # The node being visited.
        # The parent of the about-to-be visited node.
    self.tokens = tokens
        # The immutable list of input tokens.
    self.tree = tree
        # The tree of ast.AST nodes.
    #
    # Traverse the tree.
    try:
        while True:
            next(self.visitor(tree))
    except StopIteration:
        pass
#@+node:ekr.20191228152949.1: *5* tong.begin/end_visitor
def begin_visitor(self, node):
    """TONG.begin_visitor: Enter a visitor."""
    # begin_visitor and end_visitor must be paired.
    self.begin_end_stack.append(node.__class__.__name__)
    # Push the previous node.
    self.node_stack.append(self.node)
    # Update self.node *last*.
    self.node = node

def end_visitor(self, node):
    """TONG.end_visitor: Leave a visitor."""
    # begin_visitor and end_visitor must be paired.
    entry_name = self.begin_end_stack.pop()
    assert entry_name == node.__class__.__name__, (repr(entry_name), node.__class__.__name__)
    assert self.node == node, (repr(self.node), repr(node))
    # Restore self.node.
    self.node = self.node_stack.pop()
#@+node:ekr.20200129084258.21: *4* function: replace_node_and_tokens
def replace_node_and_tokens(old_node, new_node, old_tokens, new_token, token_list):
    """
    Replace old_node with new_node.
    Replace all old_tokens with a single new_token.
    """
    # Replace the tokens...
        # tokens = tokens_for_node(old_node, token_list)
        # i1 = i = tokens[0].index
        # replace_token(self.tokens[i], 'string', s)
        # replace_token(token_list[i], new_token.kind, new_token.value)
    replace_token(old_token, new_token.kind, new_token.value)
    i1 = old_token.index
    j = 1
    while j < len(tokens):
        replace_token(token_list[i1 + j], 'killed', '')
        j += 1
    # Replace the node.
        # new_node = ast.Str()
        # new_node.s = s
    replace_node(new_node, old_node)
    # Update the token.
        # token = token_list[i1]
    new_token.node = new_node
    # Update the token list.
    new_node.token_list = [token]
#@+node:ekr.20200209135643.1: *4* orange.clean_leo_nodes (black only)
def clean_leo_nodes(self):  # pragma: no cover (black)
    """
    Remove all blank lines before and after Leo @+node and @+others sentinels.
    
    This is a post pass, for last-minute cleanups.
    """

    def clean_before(i):
        """Replace blank lines before self.code_list[i] with a single newline."""
        i -= 1
        cleaned = 0
        while i > 0:
            t = self.code_list[i]
            i -= 1
            if t.kind == 'blank-lines':
                t.kind, t.value = 'killed', ''  # pragma: no cover (defensive)
            elif t.kind == 'line-end':
                if cleaned > 0:
                    t.kind, t.value = 'killed', ''  # pragma: no cover (defensive)
                cleaned += 1
            else:
                break

    def clean_after(i):
        """Replace blank lines after self.code_list[i] with a single newline."""
        i += 1
        cleaned = 0
        while i < len(self.code_list):
            t = self.code_list[i]
            i += 1
            if t.kind == 'blank-lines':
                t.kind, t.value = 'killed', ''  # pragma: no cover (defensive)
            elif t.kind == 'line-end':
                if cleaned > 0:
                    t.kind, t.value = 'killed', ''  # pragma: no cover (defensive)
                cleaned += 1
            else:
                break

    for i, t in enumerate(self.code_list):
        if t.kind == 'comment' and (
            self.node_pat.match(t.value) or
            self.at_others_pat.match(t.value)
        ):
            clean_before(i)
            clean_after(i)
#@+node:ekr.20200107165250.50: *4* orange.find_delims (never used)
def find_delims(self, tokens):
    """
    Compute the net number of each kind of delim in the given range of tokens.
    
    Return (curlies, parens, squares)
    """
    parens, curlies, squares = 0, 0, 0
    for token in tokens:
        value = token.value
        if token.kind == 'lt':
            assert value in '([{', f"Bad lt value: {token.kind} {value}"
            if value == '{':
                curlies += 1
            elif value == '(':
                parens += 1
            elif value == '[':
                squares += 1
        elif token.kind == 'rt':
            assert value in ')]}', f"Bad rt value: {token.kind} {value}"
            if value == ')':
                parens -= 1
            elif value == ']':
                squares -= 1
            elif value == '}':
                curlies += 1
    return curlies, parens, squares
#@+node:ekr.20200212072951.1: *4* TestOrange.test_bad_break
def test_bad_break(self):

    # From setup.py.
    # The total length of the ''' string should not affect line length.
    contents = r"""\
def get_semver(tag):
    if 1:
        print(
            '''*** Failed to parse Semantic Version from git tag '{0}'.
        Expecting tag name like '5.7b2', 'leo-4.9.12', 'v4.3' for releases.
        This version can't be uploaded to PyPi.org.'''.format(tag))
    return version
"""
    contents, tokens, tree = self.make_data(contents)
    # expected = self.blacken(contents).rstrip() + '\n'
    expected = contents.rstrip() + '\n'
    results = self.beautify(contents, tokens, tree)
    if results != expected:
        g.printObj(tokens)
        g.printObj(expected, tag='expected')
        g.printObj(results, tag='results')
        assert False
#@+node:ekr.20191114161840.1: *4* TR.diff
def diff(self, results_array):
    """
    Produce a diff of self.tokens vs self.results.
    
    The results array no longer exists, so this is a vestigial method.
    """
    import difflib
    import time
    ndiff = False
    if ndiff:
        # FAILS:
        #  File "C:\Users\edreamleo\Anaconda3\lib\difflib.py", line 1017, in _fancy_replace
        #  yield '  ' + aelt
        #  TypeError: can only concatenate str (not "tuple") to str
        results = results_array
        tokens = [(z.kind, z.value) for z in self.tokens]
        gen = difflib.ndiff(tokens, results)
    else:
        # Works.
        results = [f"{z.kind:>12}:{z.value}" for z in results_array]
        tokens =  [f"{z.kind:>12}:{z.value}" for z in self.tokens]
        gen = difflib.Differ().compare(tokens, results)
    t1 = time.process_time()
    diffs = list(gen)
    t2 = time.process_time()
    print(
        f"\nDiff: tokens: {len(tokens)}, results: {len(results)}, "
        f"{len(diffs)} diffs in {(t2-t1):4.2f} sec...")
    if len(diffs) < 1000:
        legend = '\n-: only in tokens, +: only in results, ?: not in either sequence!\n'
        heading = f"tx  rx  kind {'diff key kind:value':>15}"
        line    = f"=== === ==== {'===================':>15}"
        print(legend)
        print(heading)
        print(line)
        rx = tx = 0
        for i, z in enumerate(diffs):
            kind = z[0]
            if kind != '?': # A mystery.
                print(f"{tx:<3} {rx:<3} {kind!r:4} {truncate(z[1:], 80)!s}")
            if kind == ' ':
                rx, tx = rx + 1, tx + 1
            elif kind == '+':
                rx += 1
            elif kind == '-':
                tx += 1
        # print(line)
        # print(heading)
        # print(legend)
#@+node:ekr.20200729081221.1: *3* Re #1618: verbatim sentinels
#@+node:ekr.20200211190650.1: *4* TestOrange.test_verbatim_fail
def test_verbatim_fail(self):

    line_length = 40  # For testing.
    #
    # Warning: Do not put bare sentinel lines here!
    #          Doing so destroys leoAst.py!
    #
    contents = '''\
SENTverbatim
SENT+node:ekr.20090128083459.82: ADDED.
SENTverbatim
SENT@nobeautify

def run(self):
    if index2 is None: # <--- Essential
        g.es("No matching bracket.")  # #1447.
        return
SENTverbatim
SENT+node:ekr.20090128083459.82: *3* class g.PosList (deprecated)
class PosList(list):
SENTverbatim
    SENT+others
SENTverbatim
    SENT+node:ekr.20140531104908.17611: *4* PosList.ctor
    def __init__(self, c, aList=None):
        if aList is None:
            for p in c.all_positions():
                self.append(p.copy())
        else:  # <-------
            pass
'''
    contents = contents.replace('SENT', '#@')
    contents, tokens, tree = self.make_data(contents)
    expected = contents + '\n'
    results = self.beautify(contents, tokens, tree,
        max_join_line_length=line_length,
        max_split_line_length=line_length,
    )
    # Necessary.
    expected = expected.replace('#@verbatim\n', '').rstrip() + '\n'
    results = results.replace('#@verbatim\n', '')
    assert results == expected, expected_got(expected, results)
#@+node:ekr.20200728111825.1: *4* COPY (Don't Use!) at.putCodeLine (major change)
def putCodeLine(self, s, i):
    '''Put a normal code line.'''
    at = self
    # Put @verbatim sentinel if required.
    k = g.skip_ws(s, i)
    if 1: # New
        # First, see whether the line *might* look like a sentinel
        delim1 = at.startSentinelComment
        delim2 = at.endSentinelComment
        delims = (delim1, delim2)
        pat = re.compile(fr"{re.escape(delim1)}\s*@")
        if pat.match(s, k):
            if 0:
                # This will happen infrequently. Recalculate all patterns.
                x = FastAtRead(c=self.c, gnx2vnode={})
                patterns = x.get_patterns(delims)
                # See if the *entire line* matches any of the patterns.
                if any(z.match(s, i) for z in patterns):
                    g.trace('Add @verbatim', repr(line))
                    at.putSentinel('@verbatim')
            else: # Unpleasant: Escape everything that *might* be a sentinel.
                at.putSentinel('@verbatim')
    else:
        if g.match(s, k, at.startSentinelComment + '@'):
            at.putSentinel('@verbatim')
    j = g.skip_line(s, i)
    line = s[i:j]
    # Don't put any whitespace in otherwise blank lines.
    if len(line) > 1:  # Preserve *anything* the user puts on the line!!!
        if not at.raw:
            at.putIndent(at.indent, line)
        if line[-1:] == '\n':
            at.os(line[:-1])
            at.onl()
        else:
            at.os(line)
    elif line and line[-1] == '\n':
        at.onl()
    elif line:
        at.os(line)  # Bug fix: 2013/09/16
    else:
        g.trace('Can not happen: completely empty line')
#@+node:ekr.20191217092340.1: ** From leoAtFile.py
@nosearch
#@+node:ekr.20190111112432.1: *3* at.checkDirectory
def checkDirectory(self, directory):
    """Return True if directory exists or could be created."""
    at, c = self, self.c
    assert directory, g.callers()
    if g.os_path_exists(directory):
        return at.isWritable(directory)
    if c.config and c.config.create_nonexistent_directories:
        directory = c.expand_path_expression(directory)
        ok = g.makeAllNonExistentDirectories(directory)
        if not ok:
            g.error(f"did not create {directory}")
            return False
    return at.isWritable(directory)
#@+node:ekr.20190111111608.1: *3* at.checkPath
def checkPath(self, fileName):
    """Return True if we can write to the file's directory."""
    at = self
    assert g.os_path_isabs(fileName), (repr(fileName), g.callers())
    directory = g.os_path_dirname(fileName)
    if not at.checkDirectory(directory):
        return False
    if g.os_path_exists(fileName):
        return at.isWritable(fileName)
    return True
#@+node:ekr.20150602204757.1: *3* at.autoBeautify
def autoBeautify(self, p):
    """Auto beautify p's tree if allowed by settings and directives."""
    c = self.c
    try:
        if not p.isDirty():
            return
        if leoBeautify.should_kill_beautify(p):
            return
        if c.config.getBool('beautify-autobeautify'):
            leoBeautify.beautifyPythonTree(event={'c': c, 'p0': p.copy()})
    except Exception:
        g.es('unexpected exception')
        g.es_exception()
#@+node:ekr.20200108180111.1: ** From leoBeautify.py
#@+node:ekr.20190725154916.1: *3* class BlackCommand
class BlackCommand:
    """A class to run black on all Python @<file> nodes in c.p's tree."""

    # tag1 must be executable, and can't be pass.
    tag1 = "if 1: print('') # black-tag1:::"
    tag2 = ":::black-tag2"
    tag3 = "# black-tag3:::"

    def __init__(self, c):
        self.c = c
        self.wrapper = c.frame.body.wrapper
        self.reloadSettings()
    @others
#@+node:ekr.20190926105124.1: *4* black.reloadSettings
@nobeautify

def reloadSettings(self):
    c = self.c
    ### keep_comments = c.config.getBool('black-keep-comment-indentation', default=True)
    ### self.sanitizer = SyntaxSanitizer(c, keep_comments)
    self.line_length = c.config.getInt("black-line-length") or 88
    # This should be on a single line,
    # so the check-settings script in leoSettings.leo will see them.
    self.normalize_strings = c.config.getBool("black-string-normalization", default=False)
#@+node:ekr.20200103055140.1: *3* commands
#@+node:ekr.20190725154916.7: *4* black.blacken_node
def blacken_node(self, root, diff_flag, check_flag=False):
    """Run black on all Python @<file> nodes in root's tree."""
    c = self.c
    if not black or not root:
        return
    t1 = time.process_time()
    self.changed, self.errors, self.total = 0, 0, 0
    self.undo_type = 'blacken-node'
    self.blacken_node_helper(root, check_flag, diff_flag)
    t2 = time.process_time()
    if not g.unitTesting:
        print(
            f'{root.h}: scanned {self.total} node{g.plural(self.total)}, '
            f'changed {self.changed} node{g.plural(self.changed)}, '
            f'{self.errors} error{g.plural(self.errors)} '
            f'in {t2-t1:5.2f} sec.'
        )
    if self.changed or self.errors:
        c.redraw()
#@+node:ekr.20190726013924.1: *4* black.blacken_node_helper
def blacken_node_helper(self, p, check_flag, diff_flag):
    """
    blacken p.b, incrementing counts and stripping unnecessary blank lines.
    
    Return True if p.b was actually changed.
    """
    trace = 'black' in g.app.debug and not g.unitTesting
    if not should_beautify(p):
        return False
    c = self.c
    self.total += 1
    language = g.findLanguageDirectives(c, p)
    if language != 'python':
        g.trace(f"skipping node: {p.h}")
        return False
    body = p.b.rstrip() + '\n'
    comment_string, body2 = self.sanitizer.comment_leo_lines(p=p)
    try:
        # Support black, version 19.3b0.
        mode = black.FileMode()
        mode.line_length = self.line_length
        mode.string_normalization = self.normalize_strings
        # Note: format_str does not check parse trees,
        #       so in effect, it already runs in fast mode.
        body3 = black.format_str(body2, mode=mode)
    except IndentationError:
        g.warning(f"IndentationError: Can't blacken {p.h}")
        g.es_print(f"{p.h} will not be changed")
        g.printObj(body2, tag='Sanitized syntax')
        if g.unitTesting:
            raise
        p.setMarked()
        p.setDirty()
        return False
    except(SyntaxError, black.InvalidInput):
        g.warning(f"SyntaxError: Can't blacken {p.h}")
        g.es_print(f"{p.h} will not be changed")
        g.printObj(body2, tag='Sanitized syntax')
        if g.unitTesting:
            raise
        p.setMarked()
        p.setDirty()
        return False
    except Exception:
        g.warning(f"Unexpected exception: {p.h}")
        g.es_print(f"{p.h} will not be changed")
        g.printObj(body2, tag='Sanitized syntax')
        g.es_exception()
        if g.unitTesting:
            raise
        p.setMarked()
        p.setDirty()
        return False
    if trace:
        g.printObj(body2, tag='Sanitized syntax')
    result = self.sanitizer.uncomment_leo_lines(comment_string, p, body3)
    if check_flag or result == body:
        if not g.unitTesting:
            return False
    if diff_flag:
        print('=====', p.h)
        print(black.diff(body, result, "old", "new")[16:].rstrip()+'\n')
        return False
    # Update p.b and set undo params.
    self.changed += 1
    p.b = result
    c.frame.body.updateEditors()
    p.v.contentModified()
    c.undoer.setUndoTypingParams(p, 'blacken-node', oldText=body, newText=result)
    p.setDirty()
    return True
#@+node:ekr.20190729065756.1: *4* black.blacken_tree
def blacken_tree(self, root, diff_flag, check_flag=False):
    """Run black on all Python @<file> nodes in root's tree."""
    c = self.c
    if not black or not root:
        return
    t1 = time.process_time()
    self.changed, self.errors, self.total = 0, 0, 0
    undo_type = 'blacken-tree'
    bunch = c.undoer.beforeChangeTree(root)
    # Blacken *only* the selected tree.
    changed = False
    for p in root.self_and_subtree():
        if self.blacken_node_helper(p, check_flag, diff_flag):
            changed = True
    if changed:
        c.setChanged()
        c.undoer.afterChangeTree(root, undo_type, bunch)
    t2 = time.process_time()
    if not g.unitTesting:
        print(
            f'{root.h}: scanned {self.total} node{g.plural(self.total)}, '
            f'changed {self.changed} node{g.plural(self.changed)}, '
            f'{self.errors} error{g.plural(self.errors)} '
            f'in {t2-t1:5.2f} sec.'
        )
    if self.changed and not c.changed:
        c.setChanged()
    if self.changed or self.errors:
        c.redraw()
#@+node:ekr.20190830043650.1: *4* blacken-check-tree
@g.command('blkc')
@g.command('blacken-check-tree')
def blacken_check_tree(event):
    """
    Run black on all nodes of the selected tree, reporting only errors.
    """
    c = event.get('c')
    if not c:
        return
    if black:
        BlackCommand(c).blacken_tree(c.p, diff_flag=False, check_flag=True)
    else:
        g.es_print('can not import black')
#@+node:ekr.20190829163640.1: *4* blacken-diff-node
@g.command('blacken-diff-node')
def blacken_diff_node(event):
    """
    Run black on the selected node.
    """
    c = event.get('c')
    if not c:
        return
    if black:
        BlackCommand(c).blacken_node(c.p, diff_flag=True)
    else:
        g.es_print('can not import black')
#@+node:ekr.20190829163652.1: *4* blacken-diff-tree
@g.command('blkd')
@g.command('blacken-diff-tree')
def blacken_diff_tree(event):
    """
    Run black on all nodes of the selected tree,
    or the first @<file> node in an ancestor.
    """
    c = event.get('c')
    if not c:
        return
    if black:
        BlackCommand(c).blacken_tree(c.p, diff_flag=True)
    else:
        g.es_print('can not import black')
#@+node:ekr.20190725155006.1: *4* blacken-node
@g.command('blacken-node')
def blacken_node(event):
    """
    Run black on the selected node.
    """
    c = event.get('c')
    if not c:
        return
    if black:
        BlackCommand(c).blacken_node(c.p, diff_flag=False)
    else:
        g.es_print('can not import black')
#@+node:ekr.20190729105252.1: *4* blacken-tree
@g.command('blk')
@g.command('blacken-tree')
def blacken_tree(event):
    """
    Run black on all nodes of the selected tree,
    or the first @<file> node in an ancestor.
    """
    c = event.get('c')
    if not c:
        return
    if black:
        BlackCommand(c).blacken_tree(c.p, diff_flag=False)
    else:
        g.es_print('can not import black')
#@+node:ekr.20150528131012.5: *4* beautify-tree
@g.command('beautify-tree')
@g.command('pretty-print-tree')
def beautifyPythonTree(event):
    """Beautify all python files in the selected outline."""
    c = event.get('c')
    if c:
        PythonTokenBeautifier(c).beautify_tree(c.p)
#@+node:ekr.20150528131012.4: *4* beautify-node
@g.command('beautify-node')
@g.command('pretty-print-node')
def prettyPrintPythonNode(event):
    """Beautify a single Python node."""
    c = event.get('c')
    if c:
        PythonTokenBeautifier(c).beautify_node(c.p)
    
    
#@+node:ekr.20200108180155.1: *3* functions & tests
#@+node:ekr.20200110014220.2: *4* @@test leoBeautify.CPrettyPrinter
import leo.core.leoBeautify as leoBeautify
cpp = leoBeautify.CPrettyPrinter(c)
fn = 'c tokenize test'
p2 = g.findNodeInTree(c,p,fn)
assert p2,'not found: %s' % (fn)

if 1: # test of indent.
    # import os ; os.system('cls')
    cpp.indent(p2)
if 0: # test of tokenize.
    aList = cpp.tokenize(p2.b)
    assert(p2.b == ''.join(aList))
#@+node:ekr.20200110014220.3: *5* c tokenize test
@language c

static exit_values_ty indent_main_loop(void)
{
    codes_ty         hd_type         = code_eof;
    char           * t_ptr           = NULL;
    codes_ty         type_code       = start_token;
    exit_values_ty   file_exit_value = total_success;
    int              dec_ind         = 0; /* current indentation for declarations */

    BOOLEAN          scase           = false; /* true when we've just see a "case";
                                               * determines what to do with the
                                               * following colon */
    BOOLEAN          flushed_nl;              /* Used when buffering up comments to remember that
                                               * a newline was passed over */
    BOOLEAN          sp_sw           = false; /* true when in the expression part of if(...),
                                               * while(...), etc. */
    BOOLEAN          force_nl        = false;

    /* last_token_ends_sp: True if we have just encountered the end of an if (...),
     * etc. (i.e. the ')' of the if (...) was the last token).  The variable is
     * set to 2 in the middle of the main token reading loop and is decremented
     * at the beginning of the loop, so it will reach zero when the second token
     * after the ')' is read.
     */

    BOOLEAN          last_token_ends_sp = false;

    BOOLEAN          last_else = false; /* true if last keyword was an else */

    for (;;)
    {
        /* this is the main loop.  it will go until
         * we reach eof */

        BOOLEAN is_procname_definition;
        bb_code_ty can_break;

        if (type_code != newline)
        {
            can_break = parser_state_tos->can_break;
        }

        parser_state_tos->last_saw_nl = false;
        parser_state_tos->can_break = bb_none;

        type_code = lexi ();    /* lexi reads one token.  "token" points to
                                 * the actual characters. lexi returns a code
                                 * indicating the type of token */

        /* If the last time around we output an identifier or
         * a paren, then consider breaking the line here if it's
         * too long.
         *
         * A similar check is performed at the end of the loop, after
         * we've put the token on the line. */

        if ((settings.max_col > 0) &&
            (buf_break != NULL) &&
            ( ( (parser_state_tos->last_token == ident) &&
                (type_code != comma) &&
                (type_code != semicolon) &&
                (type_code != newline) &&
                (type_code != form_feed) &&
                (type_code != rparen) &&
                (type_code != struct_delim)) ||
              ( (parser_state_tos->last_token == rparen) &&
                (type_code != comma) &&
                (type_code != rparen) ) ) &&
            (output_line_length () > settings.max_col))
        {
            break_line = 1;
        }

        if (last_token_ends_sp > 0)
        {
            last_token_ends_sp--;
        }

        is_procname_definition =
                (((parser_state_tos->procname[0] != '\0') &&
                  parser_state_tos->in_parameter_declaration) ||
                 (parser_state_tos->classname[0] != '\0'));

        /* The following code moves everything following an if (), while (),
         * else, etc. up to the start of the following stmt to a buffer. This
         * allows proper handling of both kinds of brace placement.
         */

        flushed_nl = false;

        if (!search_brace(&type_code, &force_nl, &flushed_nl, &last_else, &is_procname_definition))
        {
            /* Hit EOF unexpectedly in comment. */
            return indent_punt;
        }
        
        if (type_code == code_eof)
        {
            /* we got eof */
            if (s_lab != e_lab || s_code != e_code || s_com != e_com)   /* must dump end of line */
            {
                dump_line(true, &paren_target);
            }

            if (parser_state_tos->tos > 1)      /* check for balanced braces */
            {
                ERROR (_("Unexpected end of file"), 0, 0);
                file_exit_value = indent_error;
            }

            if (settings.verbose)
            {
                printf (_("There were %d non-blank output lines and %d comments\n"),
                        (int) out_lines, (int) com_lines);
                if (com_lines > 0 && code_lines > 0)
                {
                    printf (_("(Lines with comments)/(Lines with code): %6.3f\n"),
                            (1.0 * com_lines) / code_lines);
                }
            }
            flush_output ();

            return file_exit_value;                                              /* RETURN */
        }

        if ((type_code != comment) &&
            (type_code != cplus_comment) &&
            (type_code != newline) &&
            (type_code != preesc) &&
            (type_code != form_feed))
        {
            if (force_nl &&
                (type_code != semicolon) &&
                ( (type_code != lbrace) ||
                  (!parser_state_tos->in_decl && !settings.btype_2) ||
                  (parser_state_tos->in_decl && !settings.braces_on_struct_decl_line) ||
                  (parser_state_tos->last_token == rbrace)))
            {
                if (settings.verbose && !flushed_nl)
                {
                    WARNING (_("Line broken 2"), 0, 0);
                }

                flushed_nl = false;
                dump_line(true, &paren_target);
                parser_state_tos->want_blank = false;
                force_nl = false;
            }

            parser_state_tos->in_stmt = true;   /* turn on flag which causes
                                                 * an extra level of
                                                 * indentation. this is
                                                 * turned off by a ; or } */
            if (s_com != e_com)
            {
                /* the code has an embedded comment in the
                 * line. Move it from the com buffer to the
                 * code buffer.
                 *
                 * Do not add a space before the comment if it is the first
                 * thing on the line.
                 */

                if (e_code != s_code)
                {
                    set_buf_break (bb_embedded_comment_start, paren_target);
                    *e_code++ = ' ';
                    embedded_comment_on_line = 2;
                }
                else
                {
                    embedded_comment_on_line = 1;
                }

                for (t_ptr = s_com; *t_ptr; ++t_ptr)
                {
                    check_code_size();
                    *e_code++ = *t_ptr;
                }

                set_buf_break (bb_embedded_comment_end, paren_target);
                *e_code++ = ' ';
                *e_code = '\0'; /* null terminate code sect */
                parser_state_tos->want_blank = false;
                e_com = s_com;
            }
        }
        else if ((type_code != comment) &&
                 (type_code != cplus_comment) &&
                 !(settings.break_function_decl_args &&
                   (parser_state_tos->last_token == comma)) &&
                 !( (parser_state_tos->last_token == comma) &&
                    !settings.leave_comma))
        {
            /* preserve force_nl thru a comment but
             * cancel forced newline after newline, form feed, etc.
             * however, don't cancel if last thing seen was comma-newline
             * and -bc flag is on. */

            force_nl = false;
        }

        /* Main switch on type of token scanned */

        check_code_size();
        
        /* now, decide what to do with the token */

        handle_the_token(type_code, &scase, &force_nl, &sp_sw, &flushed_nl,
                         &hd_type, &dec_ind, &last_token_ends_sp, &file_exit_value,
                         can_break, &last_else, is_procname_definition);
        
        *e_code = '\0';         /* make sure code section is null terminated */

        if ((type_code != comment) &&
            (type_code != cplus_comment) &&
            (type_code != newline) &&
            (type_code != preesc) &&
            (type_code != form_feed))
        {
            parser_state_tos->last_token = type_code;
        }

        /* Now that we've put the token on the line (in most cases),
         * consider breaking the line because it's too long.
         *
         * Don't consider the cases of `unary_op', newlines,
         * declaration types (int, etc.), if, while, for,
         * identifiers (handled at the beginning of the loop),
         * periods, or preprocessor commands. */

        if ((settings.max_col > 0) && (buf_break != NULL))
        {
            if ( ( (type_code == binary_op) ||
                   (type_code == postop) ||
                   (type_code == question) ||
                   ((type_code == colon) && (scase || (squest <= 0))) ||
                   (type_code == semicolon) ||
                   (type_code == sp_nparen) ||
                   (type_code == sp_else) ||
                   ((type_code == ident) && (*token == '\"')) ||
                   (type_code == struct_delim) ||
                   (type_code == comma)) &&
                 (output_line_length () > settings.max_col))
            {
                break_line = 1;
            }
        }
    }                           /* end of main infinite loop */
}
#@+node:ekr.20200110014220.4: *4* @@test SyntaxSanitizer
from leo.core.leoBeautify import  SyntaxSanitizer
for child in p.subtree():
    child_s = child.b
    for keep in (True, False):
        # Setup.
        sanitizer = SyntaxSanitizer(c, keep)
        comment, result1 = sanitizer.comment_leo_lines(child)
        # Test basic round-tripping.
        result2 = sanitizer.uncomment_leo_lines(comment, child, s0=result1)
        assert child_s.rstrip() == result2.rstrip(), (
            f"Round-trip FAIL: keep: {keep}, {child.h}\n"
            f"{g.objToString(child_s, tag='child_s')}\n"
            f"{g.objToString(result1, tag='result1')}\n"
            f"{g.objToString(result2, tag='result2')}")
#@+node:ekr.20200110014220.5: *5* basic test
@language python
@
This is a doc part.
@c

def spam():
    if 1:
        # Regular comment.
        print('-----')
            # Indented comment.
    else:
        pass
#@+node:ekr.20200110014220.6: *5* @others & sections references
a = 1
<< section ref >>
b = 2

if 1:
    @others

# Note: section references will fail in if statements, because pass is not valid there:
    
    # if (
        # <section ref>
    # ):
        # pass
#@+node:ekr.20200110014220.7: *5* fast_at.scan_lines
@language python

def scan_lines(self, delims, first_lines, lines, path, start, test=False):
    '''Scan all lines of the file, creating vnodes.'''
    << init scan_lines >>
    << define dump_v >>
    i = 0 # To keep pylint happy.
    for i, line in enumerate(lines[start:]):
        # Order matters.
        << 1. common code for all lines >>
        << 2. short-circuit later tests >>
        << 3. handle @others >> # clears in_doc
        << 4. handle section refs >> # clears in_doc.
        # Order doesn't matter, but match more common sentinels first.
        << handle node_start >>
        << handle end of @doc & @code parts >>
        << handle @all >>
        << handle afterref >>
        << handle @first and @last >>
        << handle @comment >>
        << handle @delims >>
        << handle @raw >>
        << handle @-leo >>
        # These must be last, in this order.
        << Last 1. handle remaining @@ lines >>
        << Last 2. handle remaining @doc lines >>
        << Last 3. handle remaining @ lines >>
    else:
        # No @-leo sentinel
        return None, []
    # Handle @last lines.
    last_lines = lines[start+i:]
    if last_lines:
        last_lines = ['@last ' + z for z in last_lines]
        gnx2body[root_gnx] = gnx2body[root_gnx] + last_lines
    self.post_pass(gnx2body, gnx2vnode, root_v)
    return root_v, last_lines
#@+node:ekr.20200110014220.8: *6* << init scan_lines >>
#
# Simple vars...
afterref = False
    # A special verbatim line follows @afterref.
clone_v = None
    # The root of the clone tree.
    # When not None, we are scanning a clone and all it's descendants.
delim_start, delim_end = delims
    # The start/end delims.
doc_skip = (delim_start + '\n', delim_end + '\n')
    # To handle doc parts.
first_i = 0
    # Index into first array.
in_doc = False
    # True: in @doc parts.
in_raw = False
    # True: @raw seen.
is_cweb = delim_start == '@q@' and delim_end == '@>'
    # True: cweb hack in effect.
indent = 0 
    # The current indentation.
level_stack = []
    # Entries are (vnode, in_clone_tree)
n_last_lines = 0
    # The number of @@last directives seen.
root_seen = False
    # False: The next +@node sentinel denotes the root, regardless of gnx.
    # Needed to handle #1065 so reads will not create spurious child nodes.
sentinel = delim_start + '@'
    # Faster than a regex!
stack = []
    # Entries are (gnx, indent, body)
    # Updated when at+others, at+<section>, or at+all is seen.
verbline = delim_start + '@verbatim' + delim_end + '\n'
    # The spelling of at-verbatim sentinel
verbatim = False
    # True: the next line must be added without change.
#
# Init the data for the root node.
#

#
# Init the parent vnode for testing.
#
if self.test:
    root_gnx = gnx = 'root-gnx'
        # The node that we are reading.
        # start with the gnx for the @file node.
    gnx_head =  '<hidden top vnode>'
        # The headline of the root node.
    context = None
    parent_v = self.VNode(context=context, gnx=gnx)
    parent_v._headString = gnx_head
        # Corresponds to the @files node itself.
else:
    # Production.
    root_gnx = gnx = self.root.gnx
    context = self.c
    parent_v = self.root.v
root_v = parent_v
    # Does not change.
level_stack.append((root_v, False),)
#
# Init the gnx dict last.
#
gnx2vnode = self.gnx2vnode
    # Keys are gnx's, values are vnodes.
gnx2body = {}
    # Keys are gnxs, values are list of body lines.
gnx2vnode[gnx] = parent_v
    # Add gnx to the keys
gnx2body[gnx] = body = first_lines
    # Add gnx to the keys.
    # Body is the list of lines presently being accumulated.
#
# get the patterns.
after_pat, all_pat, code_pat, comment_pat, delims_pat,\
doc_pat, end_raw_pat, first_pat, last_pat, \
node_start_pat, others_pat, raw_pat, ref_pat = self.get_patterns(delims)
#@+node:ekr.20200110014220.9: *6* << define dump_v >>
def dump_v():
    '''Dump the level stack and v.'''
    print('----- LEVEL', level, v.h)
    print('       PARENT', parent_v.h)
    print('[')
    for i, data in enumerate(level_stack):
        v2, in_tree = data
        print('%2s %5s %s' % (i+1, in_tree, v2.h))
    print(']')
    print('PARENT.CHILDREN...')
    g.printObj([v3.h for v3 in parent_v.children])
    print('PARENTS...')
    g.printObj([v4.h for v4 in v.parents])
#@+node:ekr.20200110014220.10: *6* << 1. common code for all lines >>
if verbatim:
    # We are in raw mode, or other special situation.
    # Previous line was verbatim sentinel. Append this line as it is.
    if afterref:
        afterref = False
        if body: # a List of lines.
            body[-1] = body[-1].rstrip() + line
        else:
            body = [line]
        verbatim = False
    elif in_raw:
        m = end_raw_pat.match(line)
        if m:
            in_raw = False
            verbatim = False
        else:
             body.append(line)
             # Continue verbatim/raw mode.
    else:
        body.append(line)
        verbatim = False
    continue
if line == verbline: # <delim>@verbatim.
    verbatim = True
    continue
#
# Strip the line only once.
strip_line = line.strip()
#
# Undo the cweb hack.
if is_cweb and line.startswith(sentinel):
    line = line[:len(sentinel)] + line[len(sentinel):].replace('@@', '@')
# Adjust indentation.
if indent and line[:indent].isspace() and len(line) > indent:
    line = line[indent:]
#@+node:ekr.20200110014220.11: *6* << 2. short-circuit later tests >>
# This is valid because all following sections are either:
# 1. guarded by 'if in_doc' or
# 2. guarded by a pattern that matches the start of the sentinel.   
#
if not in_doc and not strip_line.startswith(sentinel):
    # lstrip() is faster than using a regex!
    body.append(line)
    continue
#@+node:ekr.20200110014220.12: *6* << 3. handle @others >>
m = others_pat.match(line)
if m:
    in_doc = False
    if m.group(2) == '+': # opening sentinel
        body.append('%s@others%s\n' % (m.group(1), m.group(3) or ''))
        stack.append((gnx, indent, body))
        indent += m.end(1) # adjust current identation
    else: # closing sentinel.
        # m.group(2) is '-' because the pattern matched.
        gnx, indent, body = stack.pop()
    continue
#@+node:ekr.20200110014220.13: *6* << 4. handle section refs >>
m = ref_pat.match(line)
if m:
    in_doc = False
    if m.group(2) == '+':
        # open sentinel.
        body.append(m.group(1) + g.angleBrackets(m.group(3)) + '\n')
        stack.append((gnx, indent, body))
        indent += m.end(1)
    else:
        # close sentinel.
        # m.group(2) is '-' because the pattern matched.
        gnx, indent, body = stack.pop()
    continue
#@+node:ekr.20200110014220.14: *6* << handle node_start >>
m = node_start_pat.match(line)
if m:
    in_doc, in_raw = False, False
    gnx, head = m.group(2), m.group(5)
    level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
        # m.group(3) is the level number, m.group(4) is the number of stars.
    v = gnx2vnode.get(gnx)
    #
    # Case 1: The root @file node. Don't change the headline.
    if not root_seen:
        # Fix #1064: The node represents the root, regardless of the gnx!
        root_seen = True
        clone_v = None
        gnx2body[gnx] = body = []
        if not v:
            # Fix #1064.
            v = root_v
            # This message is annoying when using git-diff.
                # if gnx != root_gnx:
                    # g.es_print("using gnx from external file: %s" % (v.h), color='blue')
            gnx2vnode [gnx] = v
            v.fileIndex = gnx
        v.children = []
        continue
    #
    # Case 2: We are scanning the descendants of a clone.
    parent_v, clone_v = level_stack[level-2]
    if v and clone_v:
        # The last version of the body and headline wins..
        gnx2body[gnx] = body = []
        v._headString = head
        # Update the level_stack.
        level_stack = level_stack[:level-1]
        level_stack.append((v, clone_v),)
        # Always clear the children!
        v.children=[]
        parent_v.children.append(v)
        continue
    #
    # Case 3: we are not already scanning the descendants of a clone.
    if v:
        # The *start* of a clone tree. Reset the children.
        clone_v = v
        v.children = []
    else:
        # Make a new vnode.
        v = self.VNode(context=context, gnx=gnx)
    #
    # The last version of the body and headline wins.
    gnx2vnode[gnx] = v
    gnx2body[gnx] = body = []
    v._headString = head
    #
    # Update the stack.
    level_stack = level_stack[:level-1]
    level_stack.append((v, clone_v),)
    #
    # Update the links.
    assert v != root_v
    parent_v.children.append(v)
    v.parents.append(parent_v)
    # dump_v()
    continue
#@+node:ekr.20200110014220.15: *6* << handle end of @doc & @code parts >>
if in_doc:
    # When delim_end exists the doc block:
    # - begins with the opening delim, alonw on its own line
    # - ends with the closing delim, alone on its own line.
    # Both of these lines should be skipped
    if line in doc_skip:
        # doc_skip is (delim_start + '\n', delim_end + '\n')
        continue
    #
    # Check for @c or @code.
    m = code_pat.match(line)
    if m:
        in_doc = False 
        body.append('@code\n' if m.group(1) else '@c\n')
        continue
else:
    m = doc_pat.match(line)
    if m:
        # @+at or @+doc?
        doc = '@doc' if m.group(1) == 'doc' else '@'
        doc2 = m.group(2) or '' # Trailing text.
        if doc2:
            body.append('%s%s\n'%(doc, doc2))
        else:
            body.append(doc + '\n')
        # Enter @doc mode.
        in_doc = True
        continue
#@+node:ekr.20200110014220.16: *6* << handle @all >>
m = all_pat.match(line)
if m:
    # @all tells Leo's *write* code not to check for undefined sections.
    # Here, in the read code, we merely need to add it to the body.
    # Pushing and popping the stack may not be necessary, but it can't hurt.
    if m.group(2) == '+': # opening sentinel
        body.append('%s@all%s\n' % (m.group(1), m.group(3) or ''))
        stack.append((gnx, indent, body))
    else: # closing sentinel.
        # m.group(2) is '-' because the pattern matched.
        gnx, indent, body = stack.pop()
        gnx2body[gnx] = body
    continue
#@+node:ekr.20200110014220.17: *6* << handle afterref >>
m = after_pat.match(line)
if m:
    afterref = True
    verbatim = True
        # Avoid an extra test in the main loop.
    continue
#@+node:ekr.20200110014220.18: *6* << handle @first and @last >>
m = first_pat.match(line)
if m:
    if 0 <= first_i < len(first_lines):
        body.append('@first ' + first_lines[first_i])
        first_i += 1
    else:
        g.trace('\ntoo many @first lines: %s' %  path)
        print('@first is valid only at the start of @<file> nodes\n')
        g.printObj(first_lines, tag='first_lines')
        g.printObj(lines[start:i+2], tag='lines[start:i+2]')
    continue
m = last_pat.match(line)
if m:
    n_last_lines += 1
    continue
#@+node:ekr.20200110014220.19: *6* << handle @comment >>
# http://leoeditor.com/directives.html#part-4-dangerous-directives
m = comment_pat.match(line)
if m:
    # <1, 2 or 3 comment delims>
    delims = m.group(1).strip()
    # Whatever happens, retain the @delims line.
    body.append('@comment %s\n' % delims)
    delim1, delim2, delim3 = g.set_delims_from_string(delims)
        # delim1 is always the single-line delimiter.
    if delim1:
        delim_start, delim_end = delim1, ''
    else:
        delim_start, delim_end = delim2, delim3
    #
    # Within these delimiters:
    # - double underscores represent a newline.
    # - underscores represent a significant space,
    delim_start = delim_start.replace('__','\n').replace('_',' ')
    delim_end = delim_end.replace('__','\n').replace('_',' ')
    # Recalculate all delim-related values
    doc_skip = (delim_start + '\n', delim_end + '\n')
    is_cweb = delim_start == '@q@' and delim_end == '@>'
    sentinel = delim_start + '@'
    #
    # Recalculate the patterns.
    delims = delim_start, delim_end
    (
        after_pat, all_pat, code_pat, comment_pat, delims_pat,
        doc_pat, end_raw_pat, first_pat, last_pat,
        node_start_pat, others_pat, raw_pat, ref_pat
    ) = self.get_patterns(delims)
    continue
#@+node:ekr.20200110014220.20: *6* << handle @delims >>
m = delims_pat.match(line)
if m:
    # Get 1 or 2 comment delims
    # Whatever happens, retain the original @delims line.
    delims = m.group(1).strip()
    body.append('@delims %s\n' % delims)
    #
    # Parse the delims.
    delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
    m2 = delims_pat.match(delims)
    if not m2:
        g.trace('Ignoring invalid @comment: %r' % line)
        continue
    delim_start = m2.group(1)
    delim_end = m2.group(2) or ''
    #
    # Within these delimiters:
    # - double underscores represent a newline.
    # - underscores represent a significant space,
    delim_start = delim_start.replace('__','\n').replace('_',' ')
    delim_end = delim_end.replace('__','\n').replace('_',' ')
    # Recalculate all delim-related values
    doc_skip = (delim_start + '\n', delim_end + '\n')
    is_cweb = delim_start == '@q@' and delim_end == '@>'
    sentinel = delim_start + '@'
    #
    # Recalculate the patterns
    delims = delim_start, delim_end
    (
        after_pat, all_pat, code_pat, comment_pat, delims_pat,
        doc_pat, end_raw_pat, first_pat, last_pat,
        node_start_pat, others_pat, raw_pat, ref_pat
    ) = self.get_patterns(delims)
    continue
#@+node:ekr.20200110014220.21: *6* << handle @raw >>
# http://leoeditor.com/directives.html#part-4-dangerous-directives
m = raw_pat.match(line)
if m:
    in_raw = True
    verbatim = True
        # Avoid an extra test in the main loop.
    continue
#@+node:ekr.20200110014220.22: *6* << handle @-leo >>
if line.startswith(delim_start + '@-leo'):
    i += 1
    break
#@+node:ekr.20200110014220.23: *6* << Last 1. handle remaining @@ lines >>
# @first, @last, @delims and @comment generate @@ sentinels,
# So this must follow all of those.
if line.startswith(delim_start + '@@'):
    ii = len(delim_start) + 1 # on second '@'
    jj = line.rfind(delim_end) if delim_end else -1
    body.append(line[ii:jj] + '\n')
    continue
#@+node:ekr.20200110014220.24: *6* << Last 2. handle remaining @doc lines >>
if in_doc:
    if delim_end:
        # doc lines are unchanged.
        body.append(line)
    else:
        # Doc lines start with start_delim + one blank.
        body.append(line[len(delim_start)+1:])
    continue
#@+node:ekr.20200110014220.25: *6* << Last 3. handle remaining @ lines >>
# Handle an apparent sentinel line.
# This *can* happen, as the result of the git-diff command.
#
# This assert verifies the short-circuit test.
assert strip_line.startswith(sentinel), (repr(sentinel), repr(line))
#
# This trace is less important, but interesting.
g.trace(f"{g.shortFileName(self.path)}: unexpected line: {line.strip()!r}")
body.append(line)
#@+node:ekr.20150530061745.1: *4* function: main & helpers
def main():
    """External entry point for Leo's beautifier."""
    t1 = time.process_time()
    base = g.os_path_abspath(os.curdir)
    files, options = scan_options()
    for path in files:
        path = g.os_path_finalize_join(base, path)
        beautify(options, path)
    print(f'beautified {len(files)} files in {time.process_time()-t1:4.2f} sec.')
#@+node:ekr.20150601170125.1: *5* beautify (stand alone)
def beautify(options, path):
    """Beautify the file with the given path."""
    fn = g.shortFileName(path)
    s, e = g.readFileIntoString(path)
    if not s:
        return
    print(f"beautifying {fn}")
    try:
        s1 = g.toEncodedString(s)
        node1 = ast.parse(s1, filename='before', mode='exec')
    except IndentationError:
        g.warning(f"IndentationError: can't check {fn}")
        return
    except SyntaxError:
        g.warning(f"SyntaxError: can't check {fn}")
        return
    readlines = g.ReadLinesClass(s).next
    tokens = list(tokenize.generate_tokens(readlines))
    x = PythonTokenBeautifier(c=None)
    # Compute the tokens.
    s2 = x.scan_all_tokens(s, tokens)
    try:
        s2_e = g.toEncodedString(s2)
        node2 = ast.parse(s2_e, filename='before', mode='exec')
    except IndentationError:
        g.warning(f"{fn}: IndentationError in result")
        g.es_print(f"{fn} will not be changed")
        g.printObj(s2, tag='RESULT')
        return
    except SyntaxError:
        g.warning(f"{fn}: Syntax error in result")
        g.es_print(f"{fn} will not be changed")
        g.printObj(s2, tag='RESULT')
        return
    except Exception:
        g.warning(f"{fn}: Unexpected exception creating the \"after\" parse tree")
        g.es_print(f"{fn} will not be changed")
        g.es_exception()
        g.printObj(s2, tag='RESULT')
        return
    ok = leoAst.compare_asts(node1, node2)
    if not ok:
        print(f"failed to beautify {fn}")
        return
    with open(path, 'wb') as f:
        f.write(s2_e)
#@+node:ekr.20150601162203.1: *5* scan_options (stand alone)
def scan_options():
    """Handle all options. Return a list of files."""
    # This automatically implements the --help option.
    usage = "usage: python -m leo.core.leoBeautify file1, file2, ..."
    parser = optparse.OptionParser(usage=usage)
    add = parser.add_option
    add(
        '-d',
        '--debug',
        action='store_true',
        dest='debug',
        help='print the list of files and exit',
    )
    # add('-k', '--keep-blank-lines', action='store_true', dest='keep',
        # help='keep-blank-lines')
    # Parse the options.
    options, files = parser.parse_args()
    if options.debug:
        # Print the list of files and exit.
        g.trace('files...', files)
        sys.exit(0)
    return files, options
#@+node:ekr.20191028141311.1: *4* test_FstringifyTokens
def test_FstringifyTokens(c, contents,
    dump=True,
    dump_input_tokens=False,
    dump_output_tokens=False,
):
    # pylint: disable=import-self
    import tokenize
    import leo.core.leoBeautify as leoBeautify
    # Tokenize.
    tokens = list(tokenize.tokenize(io.BytesIO(contents.encode('utf-8')).readline))
    # Create a list of input tokens (BeautifierTokens).
    x = leoBeautify.FstringifyTokens(c)
    x.dump_input_tokens = dump_input_tokens
    x.dump_output_tokens = dump_output_tokens
    # Scan the input tokens, creating, a string.
    results = x.scan_all_tokens(contents, tokens)
    # Show results.
    print('')
    show(contents, 'Contents', dump=dump)
    print('')
    show(results, 'Results', dump=dump)
#@+node:ekr.20191028140946.1: *4* test_NullTokenBeautifier
def test_NullTokenBeautifier(c, contents,
    dump=True,
    dump_input_tokens=False,
    dump_output_tokens=False,
):

    # pylint: disable=import-self
    import tokenize
    import leo.core.leoBeautify as leoBeautify
    # Tokenize.
    tokens = list(tokenize.tokenize(io.BytesIO(contents.encode('utf-8')).readline))
    # Untokenize.
    x = leoBeautify.NullTokenBeautifier(c)
    x.dump_input_tokens = dump_input_tokens
    x.dump_output_tokens = dump_output_tokens
    results = x.scan_all_tokens(contents, tokens)
    # Show results.
    show(contents, 'Contents', dump=dump)
    print('')
    show(results, 'Results', dump=dump)
    return contents == results
#@+node:ekr.20191029184028.1: *4* test_PythonTokenBeautifier
def test_PythonTokenBeautifier(c, contents,
    dump=True,
    dump_input_tokens=False,
    dump_output_tokens=False,
):

    # pylint: disable=import-self
    import tokenize
    import leo.core.leoBeautify as leoBeautify
    # Create 5-tuples.
    tokens = list(tokenize.tokenize(io.BytesIO(contents.encode('utf-8')).readline))
    # Beautify.
    x = leoBeautify.PythonTokenBeautifier(c)
    x.dump_input_tokens = dump_input_tokens
    x.dump_output_tokens = dump_output_tokens
    results = x.scan_all_tokens(contents, tokens)
    # Show results.
    print('')
    show(contents, 'Contents', dump)
    print('')
    show(results, 'Results', dump)
    return results.strip() == contents.strip()
#@+node:ekr.20200110015014.2: *4* These tests use only python's tokenize module
#@+node:ekr.20200110015014.3: *5* @@test bad input order 
from tokenize import Untokenizer
u = Untokenizer()
u.prev_row = 2
u.prev_col = 2
with self.assertRaises(ValueError) as cm:
    u.add_whitespace((1,3))
self.assertEqual(cm.exception.args[0],
    'start (1,3) precedes previous end (2,2)')
# raise if previous column in row
self.assertRaises(ValueError, u.add_whitespace, (2,1))
#@+node:ekr.20200110015014.4: *5* @@test backslash continuation
from tokenize import Untokenizer
import test.test_tokenize as tt
u = Untokenizer()
u.prev_row = 1
u.prev_col =  1
u.tokens = []
u.add_whitespace((2, 0))
self.assertEqual(u.tokens, ['\\\n'])
u.prev_row = 2
u.add_whitespace((4, 4))
self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', '    '])
tt.TestRoundtrip.check_roundtrip(self, 'a\n  b\n    c\n  \\\n  c\n')
#@+node:ekr.20200110015014.5: *5* @@test iter compat
from tokenize import untokenize, Untokenizer, NAME, ENCODING

u = Untokenizer()
token = (NAME, 'Hello')
tokens = [(ENCODING, 'utf-8'), token]
u.compat(token, iter([]))
self.assertEqual(u.tokens, ["Hello "])
u = Untokenizer()
self.assertEqual(u.untokenize(iter([token])), 'Hello ')
u = Untokenizer()
self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
self.assertEqual(u.encoding, 'utf-8')
self.assertEqual(untokenize(iter(tokens)), b'Hello ')
    # *not* u.untokenize.
#@+node:ekr.20200726083127.1: *3* class FStringifyTokens(NullTokenBeautifier)
class FstringifyTokens(NullTokenBeautifier):
    """A token-based tool that converts strings containing % to f-strings."""

    undo_type = "Fstringify"

    def __init__(self, c):
        super().__init__(c)
        self.ws = ''
        self.sanitizer = SyntaxSanitizer(c, keep_comments=True)

    @others
#@+node:ekr.20200726083127.2: *4* fstring.error
def error(self, message):

    if not g.unitTesting:
        g.es_print('')
        g.es_print(f"line {self.line_number}: {message}:")
        g.es_print(self.line.strip())
#@+node:ekr.20200726083127.3: *4* fstring: Conversion
#@+node:ekr.20200726083127.4: *5* fstring.check_newlines
def check_newlines(self, tokens):
    """
    Check to ensure that no newlines appear within { and }.
    
    Return False if there is an error
    """
    level = 0
    for token in tokens:
        kind, val = token.kind, token.value
        if kind == 'op':
            if val == '{':
                level += 1
            elif val == '}':
                level -= 1
                if level < 0:
                    self.error('curly bracket underflow')
                    return False
        if '\\n' in val and level > 0:
            self.error('f-expression would contain a backslash')
            return False
    if level > 0:
        self.error('unclosed curly bracket')
        return False
    return True
#@+node:ekr.20200726083127.5: *5* fstring.clean_ws
ws_pat = re.compile(r'(\s+)([:!][0-9]\})')

def clean_ws(self, s):
    """Carefully remove whitespace before ! and : specifiers."""
    s = re.sub(self.ws_pat, r'\2', s)
    return s
#@+node:ekr.20200726083127.6: *5* fstring.compute_result
def compute_result(self, string_val, results_list):
    """
    Create the final result as follows:
        
    1. Flatten the results array.
    
    2. Using string_val (the original string) compute whether to use single
       or double quotes for the outer fstring.
    
    3. Beautify the result using the PythonTokenBeautifier class.
    
    Return the result string, or None if there are errors.
    """
    trace = False and not g.unitTesting
    # pylint: disable=import-self
    import leo.core.leoBeautify as leoBeautify
    c = self.c
    #
    # Flatten the token list.
    if trace: g.printObj(results_list, tag='TOKENS 1')
    tokens = []
    ### To do: define "flatten" helper.
    for z in results_list:
        if isinstance(z, (list, tuple)):
            tokens.extend(z)
        else:
            tokens.append(z)
    if trace: g.printObj(tokens, tag='TOKENS 2')
    #
    # Fail if the result would include a backslash within { and }.
    if not self.check_newlines(tokens):
        return None
    #
    # Ensure consistent quotes.
    ok = self.change_quotes(string_val, tokens)
    if not ok:
        if not g.unitTesting:
            self.error('string contains backslashes')
        return None
    #
    # Ensure one blank after the f-string.
    ### Doesn't work well.
    # tokens.append(self.new_token('fstringify', ' '))
    #
    # Use ptb to clean up inter-token whitespace.
    if trace: g.printObj(tokens, tag='TOKENS: before ptb')
    x = leoBeautify.PythonTokenBeautifier(c)
    x.dump_input_tokens = True
    x.dump_output_tokens = True
    result_tokens = x.scan_all_beautifier_tokens(tokens)
    #
    # Create the result.
    if trace: g.printObj(result_tokens, tag='TOKENS: after ptb')
    result = ''.join([z.to_string() for z in result_tokens])
    # Ensure a space between the new fstring and a previous name.
    if self.prev_token.kind == 'name':
        result = ' ' + result
    if self.add_trailing_ws:
        result = result + ' '
    return result
#@+node:ekr.20200726083127.7: *5* fstring.convert_fstring
def convert_fstring(self):
    """
    Scan a string, converting it to an f-string.
    The 'string' token has already been consumed.
    """
    new_token = self.new_token
    string_val = self.val
    self.add_trailing_ws = False
    specs = self.scan_format_string(string_val)
    values, tokens = self.scan_for_values()
    if len(specs) != len(values):
        self.error('Scanning error')
        g.trace('\nMISMATCH\n')
        g.trace('specs:', len(specs), 'values', len(values))
        g.printObj(specs, tag='SPECS')
        g.printObj(values, tag='VALUES')
        # Punt, without popping any more tokens.
        self.add_token('string', string_val)
        return
    # Substitute the values.
    i, results = 0, [new_token('fstringify', 'f')]
    for spec_i, m in enumerate(specs):
        value = values[spec_i]
        start, end, spec = m.start(0), m.end(0), m.group(1)
        if start > i:
            results.append(new_token('fstringify', string_val[i : start]))
        head, tail = self.munge_spec(spec)
        results.append(new_token('op', '{'))
        results.append(new_token('fstringify', value))
        if head:
            results.append(new_token('fstringify', '!'))
            results.append(new_token('fstringify', head))
        if tail:
            results.append(new_token('fstringify', ':'))
            results.append(new_token('fstringify', tail))
        results.append(new_token('op', '}'))
        i = end
    # Add the tail.
    tail = string_val[i:]
    if tail:
        results.append(new_token('fstringify', tail))
    result = self.compute_result(string_val, results)
    if result:
        # Remove whitespace before ! and :.
        result = self.clean_ws(result)
        # Actually consume the scanned tokens.
        for token in tokens:
            self.tokens.pop(0)
        self.add_token('string', result)
    else:
        # Punt, without popping any more tokens.
        self.add_token('string', string_val)
#@+node:ekr.20200726083127.8: *5* fstring.munge_spec
def munge_spec(self, spec):
    """
    Return (head, tail).
    
    The format is spec !head:tail or :tail
    
    Example specs: s2, r3
    """
    ### To do: handle more specs.
    head, tail = [], []
    if spec.startswith('+'):
        pass # Leave it alone!
    elif spec.startswith('-'):
        tail.append('>')
        spec = spec[1:]
    if spec.endswith('s'):
        spec = spec[:-1]
    if spec.endswith('r'):
        head.append('r')
        spec = spec[:-1]
    tail = ''.join(tail) + spec
    head = ''.join(head)
    return head, tail
#@+node:ekr.20200726083127.9: *5* fstring.change_quotes
def change_quotes(self, string_val, aList):
    """
    Carefully change quotes in all "inner" tokens as necessary.
    
    Return True if all went well.
    
    We expect the following "outer" tokens.
        
    aList[0]:  ('fstringify', 'f')
    aList[1]:  ('fstringify', a string starting with a quote)
    aList[-1]: ('fstringify', a string ending with a quote that matches aList[1])
    """
    # Sanity checks.
    if len(aList) < 4:
        return True
    if not string_val:
        g.es_print('no string_val!')
        return False
    delim = string_val[0]
    ### delim2 = '"' if delim == "'" else "'"
    #
    # Check tokens 0, 1 and -1.
    token0 = aList[0]
    token1 = aList[1]
    token_last = aList[-1]
    for token in token0, token1, token_last:
        if token.kind != 'fstringify':
            g.es_print(f"unexpected token: {token!r}")
            return False
    if token0.value != 'f':
        g.es_print('token[0] error!', repr(token0))
        return False
    val1 = token1.value and token1.value[0]
    if delim != val1:
        g.es_print('token[1] error!', delim, val1, repr(token1))
        return False
    val_last = token_last.value and token_last.value[-1]
    if delim != val_last:
        g.es_print('token[-1] error!', delim, val_last, repr(token_last))
        return False
    # g.printObj(aList)
    # Regularize the outer tokens.
    delim, delim2 = '"', "'"
    token1.value = delim + token1.value[1:]
    aList[1] = token1
    token_last.value = token_last.value[:-1] + delim
    aList[-1] = token_last
    #
    # Replace delim by delim2 in all inner tokens.
    for z in aList[2:-1]:
        ### g.trace(z)
        if not isinstance(z, BeautifierToken):
            g.es_print('Bad token:', repr(z))
            return False
        z.value = z.value.replace(delim, delim2)
    # g.printObj(aList)
    return True
#@+node:ekr.20200726083127.10: *5* fstring.scan_for_values
def scan_for_values(self):
    """
    Return a list of possibly parenthesized values for the format string.
    
    This method never actually consumes tokens.
    
    If all goes well, we'll skip all tokens in the tokens list.
    """
    # Skip the '%'
    new_token = self.new_token
    assert self.look_ahead(0) == ('op', '%')
    token_i, tokens = self.skip_ahead(0, 'op', '%')
    # Skip '(' if it's next
    include_paren = self.look_ahead(token_i) == ('op', '(')
    if include_paren:
        token_i, skipped_tokens = self.skip_ahead(token_i, 'op', '(')
        tokens.extend(skipped_tokens)
    # Find all tokens up to the first ')' or 'for'
    values, value_list = [], []
    while token_i < len(self.tokens):
        # Don't use look_ahead here: handle each token exactly once.
        token = self.tokens[token_i]
        token_i += 1
        tokens.append(token)
        kind, val = token.kind, token.value
        if kind == 'ws':
            continue
        if kind in ('newline', 'nl'):
            if include_paren or val.endswith('\\\n'):
                # Continue scanning, ignoring the newline.
                continue
            else:
                # The newline ends the scan.
                values.append(value_list)
                    # Retain the tokens!
                if not include_paren: # Bug fix.
                    tokens.pop()  # Rescan the ')'
                break
        if (kind, val) == ('op', ')'):
            values.append(value_list)
            if not include_paren:
                tokens.pop()  # Rescan the ')'
            break
        if (kind, val) == ('name', 'for'):
            self.add_trailing_ws = True
            tokens.pop()  # Rescan the 'for'
            values.append(value_list)
            break
        if (kind, val) == ('op', ','):
            values.append(value_list)
            value_list = []
        elif kind == 'op' and val in '([{':
            values_list2, token_i2 = self.scan_to_matching(token_i-1, val)
            value_list.extend(values_list2)
            tokens.extend(self.tokens[token_i : token_i2])
            token_i = token_i2
        elif kind == 'name':
            # Ensure separation of names.
            value_list.append(new_token(kind, val))
            value_list.append(new_token('ws', ' '))
        else:
            value_list.append(new_token(kind, val))
    return values, tokens
#@+node:ekr.20200726083127.11: *5* fstring.scan_format_string
# format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
# fill        ::=  <any character>
# align       ::=  "<" | ">" | "=" | "^"
# sign        ::=  "+" | "-" | " "
# width       ::=  integer
# precision   ::=  integer
# type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"

format_pat = re.compile(r'%(([+-]?[0-9]*(\.)?[0.9]*)*[bcdeEfFgGnoxrsX]?)')

def scan_format_string(self, s):
    """Scan the format string s, returning a list match objects."""
    result = list(re.finditer(self.format_pat, s))
    return result
#@+node:ekr.20200726083127.12: *5* fstring.scan_to_matching
def scan_to_matching(self, token_i, val):
    """
    self.tokens[token_i] represents an open (, [ or {.
    
    Return (values_list, token_i) of all tokens to the matching closing delim.
    """
    trace = False and not g.unitTesting
    new_token = self.new_token
    if trace:
        g.trace('=====', token_i, repr(val))
        g.trace(''.join([z.value for z in self.tokens[token_i:]]))
    values_list = []
    kind0, val0 = self.look_ahead(token_i)
    assert kind0 == 'op' and val0 == val and val in '([{', (kind0, val0)
    levels = [0, 0, 0]
    level_index = '([{'.index(val)
    levels[level_index] += 1
    # Move past the opening delim.
    values_list.append(new_token('op', val0))
    token_i += 1
    while token_i < len(self.tokens):
        # Don't use look_ahead here: handle each token exactly once.
        progress = token_i
        token = self.tokens[token_i]
        token_i += 1
        kind, val = token.kind, token.value
        if kind == 'ws':
            continue
        if kind in ('nl', 'newline'):
            continue
        if kind == 'op' and val in ')]}':
            values_list.append(new_token('op', val))
            level_index = ')]}'.index(val)
            levels[level_index] -= 1
            if levels == [0, 0, 0]:
                if trace:
                    g.printObj(values_list, tag=f"scan_to_matching {val!r}")
                return values_list, token_i
        elif kind == 'op' and val in '([{':
            # Recurse.
            values_list2, token_i = self.scan_to_matching(token_i-1, val)
            values_list.extend(values_list2)
        elif kind == 'name':
            # Ensure separation of names.
            values_list.append(new_token('name', val))
            values_list.append(new_token('ws', '  '))
        else:
            values_list.append(new_token(kind, val))
        assert token_i > progress, (kind, val)
    # Should never happen.
    g.trace(f"\nFAIL {token_i} {''.join(values_list)}\n")
    return [], token_i
#@+node:ekr.20200726083127.13: *4* fstring: Entries
#@+node:ekr.20200726083127.14: *5* fstring.fstringify_file
def fstringify_file(self):
    """
    Find the nearest @<file> node and convert % to fstrings within it.
    
    There is no need to sanitize code when converting an external file.
    """
    trace = True and not g.unitTesting
    verbose = False
    filename = self.find_root()
    if not filename:
        return
    # Open the file.
    with open(filename, 'r') as f:
        contents = f.read()
    if trace:
        g.trace(f"Contents...\n\n{contents}")
    # Generate tokens.
    tokens = self.tokenize_string(contents, filename)
    # Handle all tokens, creating the raw result.
    result = self.scan_all_tokens(contents, tokens)
    # Trace the results.
    changed = contents.rstrip() != result.rstrip()
    if trace and verbose:
        g.printObj(f"Code List...\n\n{self.code_list}")
    if trace:
        g.trace(f"Result...\n\n{result}")
    if not changed:
        return
    # Write the file.
    with open(filename, 'w') as f:
        f.write(result)
#@+node:ekr.20200726083127.15: *5* fstring.fstringify_node
def fstringify_node(self, p):
    """
    fstringify node p.  Return True if the node has been changed.
    """
    trace = False and not g.unitTesting
    verbose = False
    c = self.c
    if should_kill_beautify(p):
        return False
    contents = p.b
    if not contents.strip():
        return False
    # Unlike with external files, we must sanitize the text!
    comment_string, contents2 = self.sanitizer.comment_leo_lines(p=p)
    # Generate tokens.
    tokens = self.tokenize_string(contents2, p.h)
    # Handle all tokens, creating the raw result.
    raw_result = self.scan_all_tokens(contents2, tokens)
    # Undo the munging of the sources.
    result = self.sanitizer.uncomment_leo_lines(comment_string, c.p, raw_result)
    changed = contents.rstrip() != result.rstrip()
    if changed:
        p.b = result
        p.setDirty()
    # Trace the results.
    if trace and changed and verbose:
        g.trace(f"Contents...\n\n{contents}\n")
        g.trace(f"code list...\n\n{g.objToString(self.code_list)}\n")
        g.trace(f"raw result...\n\n{raw_result}\n")
        g.trace(f"Result...\n\n{result}\n")
    if trace:
        g.trace('Changed!' if changed else 'No change:', p.h)
    return changed
#@+node:ekr.20200726083127.16: *5* fstring.fstringify_tree
def fstringify_tree(self, p):
    """fstringify node p."""
    c = self.c
    if should_kill_beautify(p):
        return
    t1 = time.process_time()
    changed = total = 0
    for p in p.self_and_subtree():
        if g.scanForAtLanguage(c, p) == "python":
            total += 1
            if self.fstringify_node(p):
                changed += 1
    self.end_undo()
    if g.unitTesting:
        return
    t2 = time.process_time()
    g.es_print(
        f"scanned {total} node{g.plural(total)}, "
        f"changed {changed} node{g.plural(changed)}, "
        # f"{errors} error{g.plural(errors)} "
        f"in {t2-t1:4.2f} sec."
    )
#@+node:ekr.20200726083127.17: *4* fstring: Tokens
#@+node:ekr.20200726083127.18: *5* fstring.new_token
def new_token(self, kind, value):
    """Return a new token"""

    def item_kind(z):
        return 'string' if isinstance(z, str) else z.kind

    def val(z):
        return z if isinstance(z, str) else z.value
        
    if isinstance(value, (list, tuple)):
        return [BeautifierToken(item_kind(z), val(z)) for z in value]    
    return BeautifierToken(kind, value)
#@+node:ekr.20200726083127.19: *5* fstring.blank
def blank(self):
    """Add a blank request to the code list."""
    # Same as ptb.blank, but there is no common base class.
    prev = self.code_list[-1]
    if prev.kind not in (
        'blank',
        'blank-lines',
        'file-start',
        'line-end',
        'line-indent',
        'lt',
        'op-no-blanks',
        'unary-op',
    ):
        self.add_token('blank', ' ')
#@+node:ekr.20200726083127.20: *5* fstring.do_string
def do_string(self):
    """Handle a 'string' token."""
    # See whether a conversion is possible.
    if (
        not self.val.lower().startswith(('f', 'r'))
        and '%' in self.val
        and self.look_ahead(0) == ('op', '%')
    ):
        # Not an f or r string, and a conversion is possible.
        self.convert_fstring()
    else:
        # Just put the string
        self.add_token('string', self.val) 
#@+node:ekr.20200726083127.21: *5* fstring.do_token (override)
def do_token(self, token):
    """
    Override NullTokenBeautifier.do_token.

    Handle one input token, a BeautifierToken.
    """
    # Only the string handler is overridden.
    if token.kind == 'string':
        self.kind = token.kind
        self.val = token.value
        # Set these for error messages.
        self.line = token.line
        self.line_number = token.line_number
        self.do_string()
    else:
        # Same as super().do_token(token)
        self.code_list.append(token)
    self.prev_token = token
#@+node:ekr.20200726083127.22: *5* fstring.look_ahead & skip_ahead
def look_ahead(self, n):
    """
    Look ahead n tokens, skipping ws tokens  n >= 0.
    Return (token.kind, token.value)
    """
    while n < len(self.tokens):
        token = self.tokens[n]
        n += 1
        assert isinstance(token, BeautifierToken), (repr(token), g.callers())
        if token.kind != 'ws':
            return token.kind, token.value
    return None, None
        # Strip trailing whitespace from the token value.

def skip_ahead(self, n, target_kind, target_val):
    """
    Skip to the target token.  Only ws tokens should intervene.

    Return (n, tokens):
    """
    tokens = []
    while n < len(self.tokens):
        token = self.tokens[n]
        tokens.append(token)
        n += 1
        if (token.kind, token.value) == (target_kind, target_val):
            return n, tokens
        assert token.kind == 'ws', (token.kind, token.value)
    # Should never happen.
    return n, []
#@+node:ekr.20200726083127.23: *5* null_tok_b.scan_all_beautifier_tokens


#@+node:ekr.20150605175037.1: ** From leoCheck.py & checkerCommands.py
@first # -*- coding: utf-8 -*-
"""Experimental code checking for Leo."""
# To do:
# - Option to ignore defs without args if all calls have no args.
# * explain typical entries
import importlib
import leo.core.leoGlobals as g
import leo.core.leoAst as leoAst
importlib.reload(leoAst)
import ast
# import glob
import importlib
import os
import re
import time
@others
@language python
@tabwidth -4
@pagewidth 70
#@+node:ekr.20171207095816.1: *3* class ConventionChecker
class ConventionChecker:
    """
    A prototype of an extensible convention-checking tool.
    See: https://github.com/leo-editor/leo-editor/issues/632
    
    Here is the body of @button check-conventions:
    
        g.cls()
        if c.changed: c.save()
        
        import importlib
        import leo.core.leoCheck as leoCheck
        importlib.reload(leoCheck)
        
        fn = g.os_path_finalize_join(g.app.loadDir, '..', 'plugins', 'nodetags.py')
        leoCheck.ConventionChecker(c).check(fn=fn)
    """
    # pylint: disable=literal-comparison
        # What's wrong with `if self.test_kind is 'test'`?

    ignore = ('bool', 'dict', 'enumerate', 'list', 'tuple')
        # Things that look like function calls.

    @others
#@+node:ekr.20171210134449.1: *4* checker.Birth
def __init__(self, c):
    self.c = c
    self.class_name = None
    self.context_stack = []
        # Stack of ClassDef and FunctionDef nodes.
    # Rudimentary symbol tables...
    self.classes = self.init_classes()
    self.special_class_names = [
        'Commands', 'LeoGlobals', 'Position', 'String', 'VNode', 'VNodeBase',
    ]
    self.special_names_dict = self.init_special_names()
    # Debugging
    self.enable_trace = True
    self.file_name = None
    self.indent = 0 # For self.format.
    self.max_time = 0.0
    self.recursion_count = 0
    self.slowest_file = None
    self.stats = self.CCStats()
    # Other ivars...
    self.errors = 0
    self.line_number = 0
    self.pass_n = 0
    self.test_kind = None
    self.unknowns = {} # Keys are expression, values are (line, fn) pairs.
#@+node:ekr.20171209044610.1: *5* checker.init_classes
def init_classes(self):
    """
    Init the symbol tables with known classes.
    """
    return {
        # Pre-enter known classes.
        'Commands': {
            'ivars': {
                'p': self.Type('instance', 'Position'),
            },
            'methods': {},
        },
        'LeoGlobals': {
            'ivars': {}, # g.app, g.app.gui.
            'methods': {
                'trace': self.Type('instance', 'None')
            },
        },
        'Position': {
            'ivars': {
                'v': self.Type('instance', 'VNode'),
                'h': self.Type('instance', 'String'),
            },
            'methods': {},
        },
        'VNode': {
            'ivars': {
                'h': self.Type('instance', 'String'),
                # Vnode has no v instance!
            },
            'methods': {},
        },
        'VNodeBase': {
            'ivars': {},
            'methods': {},
        },
        'String': {
            'ivars': {},
            'methods': {}, # Possible?
        },
    }
    
#@+node:ekr.20171210133853.1: *5* checker.init_special_names
def init_special_names(self):
    """Init known special names."""
    t = self.Type
    return {
        'c': t('instance', 'Commands'),
        'c.p': t('instance', 'Position'),
        'g': t('instance', 'LeoGlobals'), # module?
        'p': t('instance', 'Position'),
        'v': t('instance', 'VNode'),
    }
#@+node:ekr.20171212015700.1: *4* checker.check & helpers (main entry)
def check(self):
    """
    The main entry point for the convention checker.

    A stand-alone version of the @button node that tested the
    ConventionChecker class.
    
    The check-conventions command in checkerCommands.py saves c and
    reloads the leoCheck module before instantiating this class and
    calling this method.
    """
    g.cls()
    c = self.c
    kind = 'production' # <----- Change only this line.
        # 'production', 'project', 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'
    join = g.os_path_finalize_join
    loadDir = g.app.loadDir
    report_stats = True
    files_table = (
        # join(loadDir, 'leoCommands.py'),
        # join(loadDir, 'leoNodes.py'),
        join(loadDir, '..', 'plugins', 'qt_tree.py'),
    )
    # ===== Don't change anything below here =====
    if kind == 'files':
        for fn in files_table:
            self.check_file(fn=fn, trace_fn=True)
    elif kind == 'production':
        for p in g.findRootsWithPredicate(c, c.p, predicate=None):
            self.check_file(fn=g.fullPath(c, p), test_kind=kind, trace_fn=True)
    elif kind in ('project', 'coverage', 'leo', 'lib2to3', 'pylint', 'rope'):
        project_name = 'leo' if kind == 'project' else kind
        self.check_project(project_name)
    elif kind == 'test':
        self.test()
    else:
        g.trace('unknown kind', repr(kind))
    if report_stats:
        self.stats.report()
#@+node:ekr.20171207100432.1: *5* checker.check_file
def check_file(self, fn=None, s=None, test_kind=None, trace_fn=False):
    """Check the contents of fn or the string s."""
    # Get the source.
    if test_kind:
        self.test_kind = test_kind
    if fn:
        sfn = g.shortFileName(fn)
        if g.os_path_exists(fn):
            s, e = g.readFileIntoString(fn)
            if s:
                s = g.toEncodedString(s, encoding=e)
            else:
                g.trace('empty file:', sfn)
                return
        else:
            g.trace('file not found:', sfn)
            return
    elif s:
        sfn = '<string>'
    else:
        g.trace('no fn or s argument')
        return
    # Check the source
    if trace_fn:
        if fn:
            print(f"===== {sfn}")
        else:
            print('===== <string>\n%s\n----- </string>\n' % s.rstrip())
    t1 = time.process_time()
    node = ast.parse(s, filename='before', mode='exec')
    self.check_helper(fn=sfn, node=node, s=s)
    t2 = time.process_time()
    t_tot = t2-t1
    if t_tot > self.max_time:
        self.max_time = t_tot
        self.slowest_file = self.file_name
#@+node:ekr.20171214150828.1: *5* checker.check_helper
def check_helper(self, fn, node, s):

    cct = self.CCTraverser(controller=self)
    for n in 1, 2:
        if self.test_kind == 'test':
            g.trace('===== PASS', n)
        # Init this pass.
        self.file_name = fn
        self.indent = 0
        self.pass_n = n
        cct.visit(node)
    self.end_file()
#@+node:ekr.20171213013004.1: *5* checker.check_project
def check_project(self, project_name):
    
    trace_fn = True
    trace_skipped = False
    self.test_kind = 'project'
    fails_dict = {
        'coverage': ['cmdline.py',],
        'lib2to3': ['fixer_util.py', 'fix_dict.py', 'patcomp.py', 'refactor.py'],
        'leo': [], # All of Leo's core files pass.
        'pylint': [
            'base.py', 'classes.py', 'format.py',
            'logging.py', 'python3.py', 'stdlib.py', 
            'docparams.py', 'lint.py',
        ],
        'rope': ['objectinfo.py', 'objectdb.py', 'runmod.py',],
    }
    fails = fails_dict.get(project_name, [])
    utils = ProjectUtils()
    files = utils.project_files(project_name, force_all=False)
    if files:
        t1 = time.process_time()
        for fn in files:
            sfn = g.shortFileName(fn)
            if sfn in fails or fn in fails:
                if trace_skipped: print('===== skipping', sfn)
            else:
                self.check_file(fn=fn, trace_fn=trace_fn)
        t2 = time.process_time()
        print('%s files in %4.2f sec. max %4.2f sec in %s' % (
            len(files), (t2-t1), self.max_time, self.slowest_file))
        if self.errors:
            print(f"{self.errors} error{g.plural(self.errors)}")
    else:
        print(f"no files for project: {project_name}")
#@+node:ekr.20171208135642.1: *5* checker.end_file & helper
def end_file(self,trace_classes=False, trace_unknowns=False):
    
    # Do *not* clear self.classes.
    self.unknowns = {}
#@+node:ekr.20171212100005.1: *6* checker.trace_unknowns
def trace_unknowns(self):
    print('----- Unknown ivars...')
    d = self.unknowns
    max_key = max([len(key) for key in d ]) if d else 2
    for key, aList in sorted(d.items()):
        # Remove duplicates that vary only in line number.
        aList2, seen = [], []
        for data in aList:
            line, fn, s = data
            data2 = (key, fn, s)
            if data2 not in seen:
                seen.append(data2)
                aList2.append(data)
        for data in aList2:
            line, fn, s = data
            print('%*s %4s %s: %s' % (
                max_key, key, line, fn, g.truncate(s, 60)))
#@+node:ekr.20171212020013.1: *5* checker.test
tests = [
'''\
class TC:
    def __init__(self, c):
        c.tc = self
    def add_tag(self, p):
        print(p.v) # AttributeError if p is a vnode.

class Test:
    def __init__(self,c):
        self.c = c
        self.tc = self.c.tc
    def add_tag(self):
        p = self.c.p
        self.tc.add_tag(p.v) # WRONG: arg should be p.
''', # comma required!
]

def test(self):

    for s in self.tests:
        s = g.adjustTripleString(s, self.c.tab_width)
        self.check_file(s=s, test_kind='test', trace_fn=True)
    if self.errors:
        print(f"{self.errors} error{g.plural(self.errors)}")
#@+node:ekr.20171216063026.1: *4* checker.error, fail, note & log_line
def error(self, node, *args, **kwargs):
    
    self.errors += 1
    print('')
    print('Error: %s' % self.log_line(node, *args, **kwargs))
    print('')
    
def fail(self, node, *args, **kwargs):
    self.stats.inference_fails += 1
    print('')
    print('Inference failure: %s' % self.log_line(node, *args, **kwargs))
    print('')
    
def log_line(self, node=None, *args, **kwargs):
    # pylint: disable=keyword-arg-before-vararg
        # putting *args first is invalid in Python 2.x.
    return 'line: %s file: %s: %s' % (
        getattr(node, 'lineno', '??'),
        self.file_name or '<string>',
        ' '.join([z if isinstance(z, str) else repr(z) for z in args]),
    )
    
def note(self, node, *args, **kwargs):

    print('')
    print('Note: %s' % self.log_line(node, *args, **kwargs))
    print('')
#@+node:ekr.20171215080831.1: *4* checker.dump, format
def dump(self, node, annotate_fields=True, level=0, **kwargs):
    """Dump the node."""
    return leoAst.AstDumper().dump(node, level=level)

def format(self, node, *args, **kwargs):
    """Format the node and possibly its descendants, depending on args."""
    s = leoAst.AstFormatter().format(node, level=self.indent, *args, **kwargs)
    return s.rstrip()
#@+node:ekr.20171208142646.1: *4* checker.resolve & helpers
def resolve(self, node, name, context, trace=False):
    """Resolve name in the given context to a Type."""
    self.stats.resolve += 1
    assert isinstance(name, str), (repr(name), g.callers())
    if context:
        if context.kind in ('error', 'unknown'):
            result = context
        elif name == 'self':
            if context.name:
                result = self.Type('instance', context.name)
            else:
                g.trace('===== NO OBJECT NAME')
                result = self.Type('error', 'no object name')
        elif context.kind in ('class', 'instance'):
            result = self.resolve_ivar(node, name, context)
        else:
            result = self.Type('error', f"unknown kind: {context.kind}")
    else:
        result = self.Type('error', f"unbound name: {name}")
    return result
#@+node:ekr.20171208134737.1: *5* checker.resolve_call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def resolve_call(self, node):
    """Resolve the head of the call's chain to a Type."""
    assert self.pass_n == 2
    self.stats.resolve_call += 1
    chain = self.get_chain(node.func)
    if chain:
        func = chain.pop()
        if isinstance(func, ast.Name):
            func = func.id
        assert isinstance(func, str), repr(func)
    if chain:
        assert isinstance(chain[0], ast.Name), repr(chain[0])
        chain[0] = chain[0].id
        # args = ','.join([self.format(z) for z in node.args])
        self.recursion_count = 0
        if self.class_name:
            context = self.Type('instance', self.class_name)
        else:
            context = self.Type('module', self.file_name)
        result = self.resolve_chain(node, chain, context)
    else:
        result = self.Type('unknown', 'empty chain')
    assert isinstance(result, self.Type), repr(result)
    return result
#@+node:ekr.20171209034244.1: *5* checker.resolve_chain
def resolve_chain(self, node, chain, context, trace=False):
    """Resolve the chain to a Type."""
    self.stats.resolve_chain += 1
    name = '<no name>'
    for obj in chain:
        name = obj.id if isinstance(obj, ast.Name) else obj
        assert isinstance(name, str), (repr(name), g.callers())
        context = self.resolve(node, name, context, trace=trace)
    assert isinstance(context, self.Type), repr(context)
    return context
#@+node:ekr.20171208173323.1: *5* checker.resolve_ivar & helpers
def resolve_ivar(self, node, ivar, context):
    """Resolve context.ivar to a Type."""
    assert self.pass_n == 2, repr(self.pass_n)
    self.stats.resolve_ivar += 1
    class_name = 'Commands' if context.name == 'c' else context.name
    self.recursion_count += 1
    if self.recursion_count > 20:
        self.report_unbounded_recursion(node, class_name, ivar, context)
        return self.Type('error', 'recursion')
    the_class = self.classes.get(class_name)
    if not the_class:
        return self.Type('error', f"no class {ivar}")
    ivars = the_class.get('ivars')
    methods = the_class.get('methods')
    if ivar == 'self':
        return self.Type('instance', class_name)
    if methods.get(ivar):
        return self.Type('func', ivar)
    if ivars.get(ivar):
        val = ivars.get(ivar)
        if isinstance(val, self.Type):
            return val
        # Check for pre-defined special names.
        for special_name, special_obj in self.special_names_dict.items():
            tail = val[len(special_name):]
            if val == special_name:
                return special_obj
            if val.startswith(special_name) and tail.startswith('.'):
                # Resovle the rest of the tail in the found context.
                return self.resolve_chain(node, tail[1:], special_obj)
        # Avoid recursion .
        head = val.split('.')
        if ivar in (val, head[0]):
            return self.Type('unknown', ivar)
        for name2 in head:
            old_context = context
            context = self.resolve(node, name2, context)
            if 0: g.trace('recursive %s: %r --> %r' % (name2, old_context, context))
        if 0: g.trace('END RECURSIVE: %r', context)
        return context
    if ivar in self.special_names_dict:
        val = self.special_names_dict.get(ivar)
        return val
    # Remember the unknown.
    self.remember_unknown_ivar(ivar)
    return self.Type('error', f"no member {ivar}")
#@+node:ekr.20171217102701.1: *6* checker.remember_unknown_ivar
def remember_unknown_ivar(self, ivar):

    d = self.unknowns
    aList = d.get(ivar, [])
    data = (self.line_number, self.file_name)
    aList.append(data)
    # tag:setter (data describing unknown ivar)
    d[ivar] = aList
    # self.error(node, 'No member:', ivar)
    return self.Type('error', 'no member %s' % ivar)
#@+node:ekr.20171217102055.1: *6* checker.report_unbounded_recursion
def report_unbounded_recursion(self, node, class_name, ivar, context):
    
    the_class = self.classes.get(class_name)
    self.error(node, 'UNBOUNDED RECURSION: %r %r\nCallers: %s' % (
        ivar, context, g.callers()))
    if 0:
        g.trace('CLASS DICT: Commands')
        g.printDict(self.classes.get('Commands'))
    if 0:
        g.trace('CLASS DICT', class_name)
        g.printDict(the_class)
#@+node:ekr.20171209065852.1: *5* checker_check_signature & helpers
def check_signature(self, node, func, args, signature):
    
    self.stats.check_signature += 1
    if signature[0] == 'self':
        signature = signature[1:]
    result = 'ok'
    for i, arg in enumerate(args):
        if i < len(signature):
            result = self.check_arg(node, func, args, arg, signature[i])
            if result == 'fail':
                self.fail(node, '\n%s(%s) incompatible with %s(%s)' % (
                    func, ','.join(args),
                    func, ','.join(signature),
                ))
                break
    if result == 'ok':
        self.stats.sig_ok += 1
    elif result == 'fail':
        self.stats.sig_fail += 1
    else:
        assert result == 'unknown'
        self.stats.sig_unknown += 1
#@+node:ekr.20171212034531.1: *6* checker.check_arg (Finish)
def check_arg(self, node, func, args, call_arg, sig_arg):
    """
    Check call_arg and sig_arg with arg (a list).
    
    To do: check keyword args.
    """
    return self.check_arg_helper(node, func, call_arg, sig_arg)

#@+node:ekr.20171212035137.1: *6* checker.check_arg_helper
def check_arg_helper(self, node, func, call_arg, sig_arg):

    special_names_dict = self.special_names_dict
    if call_arg == sig_arg or sig_arg in (None, 'None'):
        # Match anything against a default value of None.
        return 'ok'
    # Resolve the call_arg if possible.
    chain = call_arg.split('.')
    if len(chain) > 1:
        head, tail = chain[0], chain[1:]
        if head in special_names_dict:
            context = special_names_dict.get(head)
            context = self.resolve_chain(node, tail, context)
            if context.kind == 'error':
                # Caller will report the error.
                return 'unknown'
            if sig_arg in special_names_dict:
                sig_class = special_names_dict.get(sig_arg)
                return self.compare_classes(
                    node, call_arg, sig_arg, context, sig_class)
    if sig_arg in special_names_dict and call_arg in special_names_dict:
        sig_class = special_names_dict.get(sig_arg)
        call_class = special_names_dict.get(call_arg)
        return self.compare_classes(
            node, call_arg, sig_arg, call_class, sig_class)
    return 'unknown'
#@+node:ekr.20171212044621.1: *6* checker.compare_classes
def compare_classes(self, node, arg1, arg2, class1, class2):

    if class1 == class2:
        self.stats.sig_infer_ok += 1
        return 'ok'
    # The caller reports the failure.
    # self.error(node, 'FAIL', arg1, arg2, class1, class2)
    self.stats.sig_infer_fail += 1
    return 'fail'
#@+node:ekr.20171215074959.1: *4* checker.Visitors & helpers
#@+node:ekr.20171215074959.2: *5* checker.Assign & helpers
def before_Assign(self, node):
    
    s = self.format(node)
    if self.test_kind == 'test': print(s)
    if self.pass_n == 1:
        return
    self.stats.assignments += 1
    for target in node.targets:
        chain = self.get_chain(target)
        if len(chain) == 2:
            var1, var2 = chain
            assert isinstance(var1, ast.Name), repr(var1)
            assert isinstance(var2, str), repr(var2)
            name = var1.id
            if name == 'self':
                self.do_assn_to_self(node, name, var2)
            elif name in self.special_names_dict:
                self.do_assn_to_special(node, name, var2)
#@+node:ekr.20171215074959.4: *6* checker.do_assn_to_self
def do_assn_to_self(self, node, var1, var2):

    assert self.pass_n == 2
    assert var1 == 'self'
    class_name = self.class_name
    if not class_name:
        self.note(node, 'SKIP: no class name', self.format(node))
        return
    if class_name in self.special_class_names:
        # self.note(node, 'SKIP: not special', self.format(node))
        return
    d = self.classes.get(class_name)
    assert d is not None, class_name
    ivars = d.get('ivars')
    ivars[var2] = self.format(node.value)
    d['ivars'] = ivars
#@+node:ekr.20171215074959.3: *6* checker.do_assn_to_special
def do_assn_to_special(self, node, var1, var2):

    assert self.pass_n == 2
    assert var1 in self.special_names_dict, (repr(var1))
    class_name = self.class_name
    t = self.special_names_dict.get(var1)
    if not t:
        if 0: self.note(node, 'not special', var1, self.format(node).strip())
        return
    # Do not set members within the class itself.
    if t.kind == 'instance' and t.name == class_name:
        if 0: self.note(node, 'SKIP', var1, class_name)
        return
    # Resolve val, if possible.
    context = self.Type(
        'instance' if class_name else 'module',
        class_name or self.file_name,
    )
    self.recursion_count = 0
    value_s = self.format(node.value)
    resolved_type = self.resolve(node, value_s, context, trace=False)
    assert isinstance(resolved_type, self.Type), repr(resolved_type)
    if 0:
        self.note(node, f"context {context} : {value_s} ==> {resolved_type}")
    # Update var1's dict, not class_name's dict.
    d = self.classes.get(t.name)
    ivars = d.get('ivars')
    # tag:setter ivar1.ivar2 = Type
    ivars[var2] = resolved_type
    d['ivars'] = ivars
#@+node:ekr.20171215074959.5: *5* checker.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def before_Call(self, node):

    if self.test_kind == 'test':
        print(self.format(node))
    if self.pass_n == 1:
        return
    self.stats.calls += 1
    context = self.resolve_call(node)
    assert isinstance(context, self.Type)
    if context.kind != 'instance':
        return
    instance = self.classes.get(context.name)
    if not instance:
        return
    chain = self.get_chain(node.func)
    func = chain[-1]
    d = instance.get('methods')
    signature = d.get(func)
    if not signature:
        return
    if isinstance(signature, self.Type):
        pass # Already checked?
    else:
        args = [self.format(z) for z in node.args]
        signature = signature.split(',')
        self.check_signature(node, func, args, signature)
#@+node:ekr.20171215074959.7: *5* checker.ClassDef
def before_ClassDef(self, node):

    s = self.format(node, print_body=False)
    if self.test_kind == 'test': print(s)
    self.indent += 1
    self.context_stack.append(node)
    self.class_name = name = node.name
    if self.pass_n == 1:
        self.stats.classes += 1
        if name not in self.special_class_names:
            # tag:setter Init the class's dict.
            self.classes [name] = {'ivars': {}, 'methods': {}}

def after_ClassDef(self, node):

    self.indent -= 1
    if 0 and self.pass_n == 1:
        g.trace(node, self.show_stack())
        print(f"----- END class {self.class_name}. class dict...")
        g.printDict(self.classes.get(self.class_name))
    #
    # This code must execute in *both* passes.
    top = self.context_stack.pop()
    assert node == top, (node, top)
    # Set the class name
    self.class_name = None
    for node2 in reversed(self.context_stack):
        if isinstance(node2, ast.ClassDef):
            self.class_name = node2.name
            break
#@+node:ekr.20171215074959.9: *5* checker.FunctionDef
def before_FunctionDef(self, node):

    s = self.format(node, print_body=False)
    if self.test_kind == 'test': print(s)
    self.indent += 1
    self.context_stack.append(node)
    if self.pass_n == 1:
        self.stats.defs += 1
        if self.class_name not in self.special_class_names:
            if self.class_name in self.classes:
                the_class = self.classes.get(self.class_name)
                methods = the_class.get('methods')
                # tag:setter function-name=stringized-args
                methods [node.name] = self.format(node.args)
            # This is not an error.
            # else: g.error(node 'no class', node.name)

def after_FunctionDef(self, node):

    self.indent -= 1
    top = self.context_stack.pop()
    assert node == top, (node, top)
#@+node:ekr.20171216110107.1: *5* checker.get_chain
def get_chain(self,node):
    """Scan node for a chain of names."""
    chain, node1 = [], node
    while not isinstance(node, ast.Name):
        if isinstance(node, ast.Attribute):
            assert isinstance(node.attr, str), repr(node.attr)
            chain.append(node.attr)
            node = node.value
        else:
            name = node.__class__.__name__
            if name not in (
                'BoolOp', # c.config.getString('stylesheet') or ''.strip
                'Call', # c1.rootPosition().h = whatever
                'Dict', # {}.whatever.
                'Subscript', # d[x] = whatever
                'Str', # ''.join(), etc
                'Tuple', # (hPos,vPos) = self.getScroll()
            ):
                self.note(node1, '(get_chain) target %s:\n%s' % (
                    name, self.format(node1)))
            return []
    if isinstance(node, ast.Name):
        chain.append(node)
        return list(reversed(chain))
    return []
#@+node:ekr.20171215082648.1: *5* checker.show_stack
def show_stack(self):

    return g.listToString([
        '%15s %s' % (node.__class__.__name__, node.name)
            for node in self.context_stack
        ])
#@+node:ekr.20171212101613.1: *4* class CCStats
class CCStats:
    """
    A basic statistics class.  Use this way:
        
        stats = Stats()
        stats.classes += 1
        stats.defs += 1
        stats.report()
    """
    # Big sigh: define these to placate pylint.
    assignments = 0
    calls = 0
    check_signature = 0
    classes = 0
    defs = 0
    inference_fails = 0
    resolve = 0
    resolve_call = 0
    resolve_chain = 0
    resolve_ivar = 0
    sig_fail = 0
    sig_infer_fail = 0
    sig_infer_ok = 0
    sig_ok = 0
    sig_unknown = 0
        
    def report(self):
        aList = [z for z in dir(self) if not z.startswith('_') and z != 'report']
        n = max([len(z) for z in aList])
        for ivar in aList:
            print('%*s: %s' % (n, ivar, getattr(self, ivar)))
    
#@+node:ekr.20171214151001.1: *4* class CCTraverser (AstFullTraverser)
class CCTraverser (leoAst.AstFullTraverser):
    
    """A traverser class that *only* calls controller methods."""

    def __init__(self, controller):

        super().__init__()
        self.cc = controller
    
    def visit(self, node):
        """
        Visit a *single* ast node.
        Visitors are responsible for visiting children!
        """
        name = node.__class__.__name__
        assert isinstance(node, ast.AST), repr(node)
        before_method = getattr(self.cc, 'before_'+name, None)
        if before_method:
            before_method(node)
        do_method = getattr(self, 'do_'+name, None)
        do_method(node)
        after_method = getattr(self.cc, 'after_'+name, None)
        if after_method:
            after_method(node)
#@+node:ekr.20171209030742.1: *4* class Type
class Type:
    """A class to hold all type-related data."""

    kinds = ('error', 'class', 'func', 'instance', 'module', 'unknown')
    
    def __init__(self, kind, name, source=None, tag=None):

        assert kind in self.kinds, repr(kind)
        self.kind = kind
        self.name=name
        self.source = source
        self.tag = tag
        
    def __repr__(self):

        return f"<{self.kind}: {self.name}>"
        
    def __eq__(self, other):
        
        return self.kind == other.kind and self.name == other.name
#@+node:ekr.20160109102859.1: *3* class Context
class Context:
    """
    Context class (NEW)

    Represents a binding context: module, class or def.

    For any Ast context node N, N.cx is a reference to a Context object.
    """
    @others
#@+node:ekr.20160109103533.1: *4* Context.ctor
def __init__ (self, fn, kind, name, node, parent_context):
    """Ctor for Context class."""
    self.fn = fn
    self.kind = kind
    self.name = name
    self.node = node
    self.parent_context = parent_context
    # Name Data...
    self.defined_names = set()
    self.global_names = set()
    self.imported_names = set()
    self.nonlocal_names = set() # To do.
    self.st = {}
        # Keys are names seen in this context, values are defining contexts.
    self.referenced_names = set()
    # Node lists. Entries are Ast nodes...
    self.inner_contexts_list = []
    self.minor_contexts_list = []
    self.assignments_list = []
    self.calls_list = []
    self.classes_list = []
    self.defs_list = []
    self.expressions_list = []
    self.returns_list = []
    self.statements_list = []
    self.yields_list = []
    # Add this context to the inner context of the parent context.
    if parent_context:
        parent_context.inner_contexts_list.append(self)
#@+node:ekr.20160109134527.1: *4* Context.define_name
def define_name(self, name):
    """Define a name in this context."""
    self.defined_names.add(name)
    if name in self.referenced_names:
        self.referenced_names.remove(name)
#@+node:ekr.20160109143040.1: *4* Context.global_name
def global_name(self, name):
    """Handle a global name in this context."""
    self.global_names.add(name)
    # Not yet.
        # Both Python 2 and 3 generate SyntaxWarnings when a name
        # is used before the corresponding global declarations.
        # We can make the same assumpution here:
        # give an *error* if an STE appears in this context for the name.
        # The error indicates that scope resolution will give the wrong result.
        # e = cx.st.d.get(name)
        # if e:
            # self.u.error(f"name {name!r} used prior to global declaration")
            # # Add the name to the global_names set in *this* context.
            # # cx.global_names.add(name)
        # # Regardless of error, bind the name in *this* context,
        # # using the STE from the module context.
        # cx.st.d[name] = module_e
#@+node:ekr.20160109144139.1: *4* Context.import_name
def import_name(self, module, name):

    if True and name == '*':
        g.trace('From x import * not ready yet')
    else:
        self.imported_names.add(name)
#@+node:ekr.20160109145526.1: *4* Context.reference_name
def reference_name(self, name):

    self.referenced_names.add(name)
#@+node:ekr.20160108105958.1: *3* class Pass1 (AstFullTraverser)
class Pass1 (leoAst.AstFullTraverser): # V2

    """ Pass1 does the following:

    1. Creates Context objects and injects them into the new_cx field of
       ast.Class, ast.FunctionDef and ast.Lambda nodes.

    2. Calls the following Context methods: cx.define/global/import/reference_name.
       These methods update lists used later to bind names to objects.
    """
    # pylint: disable=no-member
        # Stats class defines __setattr__
        # This is a known limitation of pylint.

    @others
#@+node:ekr.20160108105958.2: *4*  p1.ctor
def __init__(self, fn):

    super().__init__()
    self.fn = fn
    # Abbreviations...
    self.stats = Stats()
    self.u = ProjectUtils()
    self.format = leoAst.AstFormatter.format
    # Present context...
    self.context = None
    self.in_attr = False
        # True: traversing inner parts of an AST.Attribute tree.
    self.module_context = None
    self.parent = None
#@+node:ekr.20160108105958.3: *4*  p1.run (entry point)
def run (self,root):

    self.visit(root)
#@+node:ekr.20160109125654.1: *4*  p1.visit
def visit(self, node):
    """Visit a *single* ast node.  Visitors are responsible for visiting children!"""
    assert isinstance(node, ast.AST), node.__class__.__name__
    # Visit the children with the new parent.
    old_parent = self.parent
    self.parent = node
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    method(node)
    self.parent = old_parent
#@+node:ekr.20160108105958.11: *4* p1.visitors
#@+node:ekr.20160109134854.1: *5* Contexts
#@+node:ekr.20160108105958.8: *6* p1.def_args_helper
# arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults)

def def_args_helper (self,cx,node):

    assert self.kind(node) == 'arguments'
    self.visit_list(node.args)
    self.visit_list(node.defaults)
    for field in ('vararg','kwarg'): # node.field is a string.
        name = getattr(node,field,None)
        if name:
            # e = cx.st.define_name(name)
            self.stats.n_param_names += 1
#@+node:ekr.20160108105958.16: *6* p1.ClassDef
# ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)

def do_ClassDef (self,node):

    # pylint: disable=arguments-differ
    old_cx = self.context
    name = node.name
    # Define the class name in the old context.
    old_cx.define_name(name)
    # Visit bases in the old context.
    # bases = self.visit_list(node.bases)
    new_cx = Context(
        fn=None,
        kind='class',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx)
    # Visit the body in the new context.
    self.context = new_cx
    self.visit_list(node.body)
    self.context = old_cx
    # Stats.
    old_cx.classes_list.append(new_cx)
#@+node:ekr.20160108105958.19: *6* p1.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#    expr? returns)

def do_FunctionDef (self,node):
    # pylint: disable=arguments-differ
    # Define the function/method name in the old context.
    old_cx = self.context
    name = node.name
    old_cx.define_name(name)
    # Create the new context.
    new_cx = Context(
        fn=None,
        kind='def',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx) # Bug fix.
    # Visit in the new context...
    self.context = new_cx
    self.def_args_helper(new_cx,node.args)
    self.visit_list(node.body)
    self.context = old_cx
    # Stats
    old_cx.defs_list.append(new_cx)
#@+node:ekr.20160108105958.23: *6* p1.Interactive
def do_Interactive(self,node):

    assert False,'Interactive context not supported'
#@+node:ekr.20160108105958.24: *6* p1.Lambda
def do_Lambda (self,node):

    # Synthesize a lambda name in the old context.
    # This name must not conflict with split names of the form name@n.
    old_cx = self.context
    name = f"Lambda@@{self.stats.n_lambdas}"
    # Define a Context for the 'lambda' variables.
    new_cx = Context(
        fn=None,
        kind='lambda',
        name=name,
        node=node,
        parent_context=old_cx)
    setattr(node,'new_cx',new_cx)
    # Evaluate expression in the new context.
    self.context = new_cx
    self.def_args_helper(new_cx,node.args)
    self.visit(node.body)
    self.context = old_cx
    # Stats...
    self.stats.n_lambdas += 1
#@+node:ekr.20160108105958.26: *6* p1.Module
def do_Module (self,node):

    # Not yet: Get the module context from the global dict if possible.
    new_cx = Context(
        fn=self.fn,
        kind='module',
        name=None,
        node=node,
        parent_context=None)
    self.context = new_cx
    self.visit_list(node.body)
    self.context = None
#@+node:ekr.20160109135022.1: *5* Expressions
#@+node:ekr.20160108105958.13: *6* p1.Attribute (Revise)
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self,node):

    # Visit...
    # cx = self.context
    old_attr, self.in_attr = self.in_attr, True
    # ctx = self.kind(node.ctx)
    self.visit(node.value)
    # self.visit(node.ctx)
    self.in_attr = old_attr
    if not self.in_attr:
        base_node = node
        kind = self.kind(base_node)
        if kind in ('Builtin','Name'):
            # base_name = base_node.id
            pass
        elif kind in ('Dict','List','Num','Str','Tuple',):
            pass
        elif kind in ('BinOp','UnaryOp'):
            pass
        else:
            assert False,kind
    # Stats...
    self.stats.n_attributes += 1
#@+node:ekr.20160108105958.17: *6* p1.Expr
# Expr(expr value)

def do_Expr(self,node):

    # Visit...
    cx = self.context
    self.visit(node.value)
    # Stats...
    self.stats.n_expressions += 1
    cx.expressions_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.27: *6* p1.Name (REWRITE)
def do_Name(self,node):

    cx  = self.context
    ctx = self.kind(node.ctx)
    name = node.id
    # def_flag,ref_flag=False,False

    if ctx in ('AugLoad','AugStore','Load'):
        # Note: AugStore does *not* define the symbol.
        cx.reference_name(name)
        self.stats.n_load_names += 1
    elif ctx == 'Store':
        # if name not in cx.global_names:
        self.stats.n_store_names += 1
    elif ctx == 'Param':
        self.stats.n_param_refs += 1
    else:
        assert ctx == 'Del',ctx
        self.stats.n_del_names += 1
#@+node:ekr.20160109140648.1: *5* Imports
#@+node:ekr.20160108105958.21: *6* p1.Import
@ From Guido:

import x            -->  x = __import__('x')
import x as y       -->  y = __import__('x')
import x.y.z        -->  x = __import__('x.y.z')
import x.y.z as p   -->  p = __import__('x.y.z').y.z
@c

def do_Import(self,node):
    """
    Add the imported file to u.files_list if needed
    and create a context for the file.
    """
    cx = self.context
    cx.statements_list.append(node)
    # e_list, names = [],[]
    for fn,asname in self.get_import_names(node):
        self.resolve_import_name(fn)
        # Not yet.
        # # Important: do *not* analyze modules not in the files list.
        # if fn2:
            # mname = self.u.module_name(fn2)
            # if g.shortFileName(fn2) in self.u.files_list:
                # if mname not in self.u.module_names:
                    # self.u.module_names.append(mname)
            # def_name = asname or mname
            # names.append(def_name)
            # e = cx.st.define_name(def_name) # sets e.defined.
            # cx.imported_symbols_list.append(def_name)
            # e_list.append(e)

            # # Add the constant type to the list of types for the *variable*.
            # mod_cx = self.u.modules_dict.get(fn2) or LibraryModuleContext(self.u,fn2)
            # e.types_cache[''] = mod_cx.module_type
            # # self.u.stats.n_imports += 1

    # for e in e_list:
        # e.defs_list.append(node)
        # e.refs_list.append(node)
#@+node:ekr.20160108105958.22: *6* p1.ImportFrom
@ From Guido:

from p.q import x       -->  x = __import__('p.q', fromlist=['x']).x
from p.q import x as y  -->  y = __import__('p.q', fromlist=['x']).x
from ..x.y import z     -->  z = __import('x.y', level=2, fromlist=['z']).z

All these equivalences are still somewhat approximate; __import__
isn't looked up the way other variables are looked up (it is taken
from the current builtins), and if the getattr operation in the "from"
versions raises AttributeError that is translated into ImportError.

There's also a subtlety where "import x.y" implies that y must be a
submodule/subpackage of x, whereas in "from x import y" it may be
either a submodule/subpackage or a plain attribute (e.g. a class,
function or some other variable).
@c

def do_ImportFrom(self,node):
    """
    Add the imported file to u.files_list if needed
    and add the imported symbols to the *present* context.
    """
    cx = self.context
    cx.statements_list.append(node)
    self.resolve_import_name(node.module)
    for fn,asname in self.get_import_names(node):
        fn2 = asname or fn
        cx.import_name(fn2)
#@+node:ekr.20160108105958.9: *6* p1.get_import_names
def get_import_names (self,node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:

        if self.kind(ast2) == 'alias':
            data = ast2.name,ast2.asname
            result.append(data)
        else:
            g.trace('unsupported kind in Import.names list',self.kind(ast2))
    return result
#@+node:ekr.20160108105958.10: *6* p1.resolve_import_name
def resolve_import_name (self,spec):
    """Return the full path name corresponding to the import spec."""
    if not spec:
        return ''
    # This may not work for leading dots.
    aList = spec.split('.')
    path = None
    # paths = None
    name = 'no name'
    for name in aList:
        try:
            pass
            ### Not ready. Old code:
                # f,path,description = imp.find_module(name,paths)
                # if not path: break
                # paths = [path]
                # if f: f.close()
        except ImportError:
            # Important: imports can fail due to Python version.
            # Thus, such errors are not necessarily searious.
            path = None
            break
    if not path:
        return ''
    if path.endswith('.pyd'):
        return ''
    return path
#@+node:ekr.20160108105958.29: *5* Operators... To be deleted???
# operator = Add | BitAnd | BitOr | BitXor | Div
# FloorDiv | LShift | Mod | Mult | Pow | RShift | Sub |

def do_Add(self,node):       setattr(node,'op_name','+')
def do_BitAnd(self,node):    setattr(node,'op_name','&')
def do_BitOr(self,node):     setattr(node,'op_name','|')
def do_BitXor(self,node):    setattr(node,'op_name','^')
def do_Div(self,node):       setattr(node,'op_name','/')
def do_FloorDiv(self,node):  setattr(node,'op_name','//')
def do_LShift(self,node):    setattr(node,'op_name','<<')
def do_Mod(self,node):       setattr(node,'op_name','%')
def do_Mult(self,node):      setattr(node,'op_name','*')
def do_Pow(self,node):       setattr(node,'op_name','**')
def do_RShift(self,node):    setattr(node,'op_name','>>')
def do_Sub(self,node):       setattr(node,'op_name','-')

# boolop = And | Or
def do_And(self,node):       setattr(node,'op_name',' and ')
def do_Or(self,node):        setattr(node,'op_name',' or ')

# cmpop = Eq | Gt | GtE | In |
# Is | IsNot | Lt | LtE | NotEq | NotIn
def do_Eq(self,node):        setattr(node,'op_name','==')
def do_Gt(self,node):        setattr(node,'op_name','>')
def do_GtE(self,node):       setattr(node,'op_name','>=')
def do_In(self,node):        setattr(node,'op_name',' in ')
def do_Is(self,node):        setattr(node,'op_name',' is ')
def do_IsNot(self,node):     setattr(node,'op_name',' is not ')
def do_Lt(self,node):        setattr(node,'op_name','<')
def do_LtE(self,node):       setattr(node,'op_name','<=')
def do_NotEq(self,node):     setattr(node,'op_name','!=')
def do_NotIn(self,node):     setattr(node,'op_name',' not in ')

# unaryop = Invert | Not | UAdd | USub
def do_Invert(self,node):   setattr(node,'op_name','~')
def do_Not(self,node):      setattr(node,'op_name',' not ')
def do_UAdd(self,node):     setattr(node,'op_name','+')
def do_USub(self,node):     setattr(node,'op_name','-')
#@+node:ekr.20160109134929.1: *5* Minor contexts
#@+node:ekr.20160109130719.1: *6* p1.comprehension (to do)
# comprehension (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):

    # Visit...
    self.visit(node.target) # A name.
    self.visit(node.iter) # An attribute.
    for z in node.ifs:
        self.visit(z)
#@+node:ekr.20160108105958.18: *6* p1.For
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self,node):

    # Visit...
    cx = self.context
    self.visit(node.target)
    self.visit(node.iter)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    # Stats...
    self.stats.n_fors += 1
    cx.statements_list.append(node)
    cx.assignments_list.append(node)
#@+node:ekr.20160108105958.30: *6* p1.With
def do_With(self,node):

    # Visit...
    cx = self.context
    self.visit(node.context_expr)
    if node.optional_vars:
        self.visit(node.optional_vars)
    for z in node.body:
        self.visit(z)
    # Stats...
    self.stats.n_withs += 1
    cx.statements_list.append(node)
#@+node:ekr.20160109135003.1: *5* Statements
#@+node:ekr.20160108105958.12: *6* p1.Assign
def do_Assign(self,node):

    # Visit...
    for z in node.targets:
        self.visit(z)
    self.visit(node.value)
    # Stats...
    cx = self.context
    self.stats.n_assignments += 1
    cx.assignments_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.14: *6* p1.AugAssign
# AugAssign(expr target, operator op, expr value)

def do_AugAssign(self,node):

    # Visit...
    self.visit(node.target)
    self.visit(node.value)
    # Stats...
    cx = self.context
    self.stats.n_assignments += 1
    cx.assignments_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20160108105958.15: *6* p1.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self,node):

    # Visit...
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
    # Stats...
    cx = self.context
    self.stats.n_calls += 1
    cx.calls_list.append(node)
#@+node:ekr.20160108105958.20: *6* p1.Global
def do_Global(self,node):

    # Visit
    cx = self.context
    for name in node.names:
        cx.global_name(name)
    # Stats...
    cx.statements_list.append(node)
    self.stats.n_globals += 1
#@+node:ekr.20160108105958.28: *6* p1.Return
def do_Return(self,node):

    # Visit...
    if node.value:
        self.visit(node.value)
    # Stats...
    self.stats.n_returns += 1
    cx = self.context
    cx.returns_list.append(node)
    cx.statements_list.append(node)
#@+node:ekr.20150525123715.1: *3* class ProjectUtils
class ProjectUtils:
    """A class to compute the files in a project."""
    # To do: get project info from @data nodes.
    @others
#@+node:ekr.20150525123715.2: *4* pu.files_in_dir
def files_in_dir(self, theDir, recursive=True, extList=None, excludeDirs=None):
    """
    Return a list of all Python files in the directory.
    Include all descendants if recursiveFlag is True.
    Include all file types if extList is None.
    """
    # import glob
    import os
    # if extList is None: extList = ['.py']
    if excludeDirs is None: excludeDirs = []
    result = []
    if recursive:
        for root, dirs, files in os.walk(theDir):
            for z in files:
                fn = g.os_path_finalize_join(root, z)
                junk, ext = g.os_path_splitext(fn)
                if not extList or ext in extList:
                    result.append(fn)
            if excludeDirs and dirs:
                for z in dirs:
                    if z in excludeDirs:
                        dirs.remove(z)
    else:
        for ext in extList:
            result.extend(g.glob_glob(f"{theDir}.*{ext}"))
    return sorted(list(set(result)))
#@+node:ekr.20150525123715.3: *4* pu.get_project_directory
def get_project_directory(self, name):
    # Ignore everything after the first space.
    i = name.find(' ')
    if i > -1:
        name = name[: i].strip()
    leo_path, junk = g.os_path_split(__file__)
    d = {
        # Change these paths as required for your system.
        'coverage': r'C:\Python26\Lib\site-packages\coverage-3.5b1-py2.6-win32.egg\coverage',
        'leo': r'C:\leo.repo\leo-editor\leo\core',
        'lib2to3': r'C:\Python26\Lib\lib2to3',
        'pylint': r'C:\Python26\Lib\site-packages\pylint',
        'rope': r'C:\Python26\Lib\site-packages\rope-0.9.4-py2.6.egg\rope\base',
        'test': g.os_path_finalize_join(g.app.loadDir, '..', 'test-proj'),
    }
    dir_ = d.get(name.lower())
    if not dir_:
        g.trace(f"bad project name: {name}")
    if not g.os_path_exists(dir_):
        g.trace('directory not found:' % (dir_))
    return dir_ or ''
#@+node:ekr.20171213071416.1: *4* pu.leo_core_files
def leo_core_files(self):
    """Return all the files in Leo's core."""
    loadDir = g.app.loadDir
    # Compute directories.
    commands_dir = g.os_path_finalize_join(loadDir, '..', 'commands')
    plugins_dir = g.os_path_finalize_join(loadDir, '..', 'plugins')
    # Compute files.
    core_files = g.glob_glob('%s%s%s' % (loadDir, os.sep, '*.py'))
    for exclude in ['format-code.py',]:
        core_files = [z for z in core_files if not z.endswith(exclude)]
    command_files = g.glob_glob(f"{commands_dir}{os.sep}{'*.py'}")
    plugins_files = g.glob_glob(f"{plugins_dir}{os.sep}{'qt_*.py'}")
    # Compute the result.
    files = core_files + command_files + plugins_files
    files = [z for z in files if not z.endswith('__init__.py')]
    return files
#@+node:ekr.20150525123715.4: *4* pu.project_files
@nobeautify

def project_files(self, name, force_all=False):
    """Return a list of all files in the named project."""
    # Ignore everything after the first space.
    i = name.find(' ')
    if i > -1:
        name = name[: i].strip()
    leo_path, junk = g.os_path_split(__file__)
    if name == 'leo':
        # Get the leo files directly.
        return self.leo_core_files()
    # Import the appropriate module.
    try:
        m = importlib.import_module(name, name)
        theDir = g.os_path_dirname(m.__file__)
    except ImportError:
        g.trace('package not found', name)
        return []
    d = {
        'coverage': (['.py'], ['.bzr', 'htmlfiles']),
        'lib2to3':  (['.py'], ['tests']),
        'pylint':   (['.py'], ['.bzr', 'test']),
        'rope':     (['.py'], ['.bzr']),
    }
    data = d.get(name.lower())
    if not data:
        g.trace(f"bad project name: {name}")
        return []
    extList, excludeDirs = data
    files = self.files_in_dir(theDir,
        recursive=True,
        extList=extList,
        excludeDirs=excludeDirs,
    )
    if files:
        if g.app.runningAllUnitTests and len(files) > 1 and not force_all:
            return [files[0]]
    if not files:
        g.trace(f"no files found for {name} in {theDir}")
    if g.app.runningAllUnitTests and len(files) > 1 and not force_all:
        return [files[0]]
    return files
#@+node:ekr.20171213155537.1: *3* class NewShowData
class NewShowData:
    """The driver class for analysis project."""
    assigns_d = {}
    calls_d = {}
    classes_d = {}
    defs_d = {}
    returns_d = {}

    @others
#@+node:ekr.20171213160214.1: *4* sd.analyze
def analyze(self, fn, root):
    
    ast_d = {
        ast.Assign: self.assigns_d,
        ast.AugAssign: self.assigns_d,
        ast.Call: self.calls_d,
        ast.ClassDef: self.classes_d,
        ast.FunctionDef: self.defs_d,
        ast.Return: self.returns_d, 
    }
    fn = g.shortFileName(fn)
    for d in ast_d.values():
        d[fn] = []
    for node in ast.walk(root):
        d = ast_d.get(node.__class__)
        if d is not None:
            d[fn].append(self.format(node))
#@+node:ekr.20171214040822.1: *4* sd.dump
def dump(self, fn, root):
    
    suppress = [
        'arg', 'arguments', 'comprehension', 'keyword',
        'Attribute', 'BinOp', 'BoolOp', 'Dict', 'IfExp', 'Index',
        'Load', 'List', 'ListComp', 'Name', 'NameConstant', 'Num',
        'Slice', 'Store', 'Str', 'Subscript', 'Tuple', 'UnaryOp',
    ]
    # statements = ['Assign', 'AugAssign', 'Call', 'Expr', 'If', 'Return',]
    errors = set()
    fn = g.shortFileName(fn)
    for node in ast.walk(root):
        name = node.__class__.__name__
        if name not in suppress:
            try:
                print('%15s: %s' % (name, self.format(node,strip=False)))
            except AttributeError:
                errors.add(name)
    g.trace('errors', sorted(errors))
    # g.printList(sorted(errors))
#@+node:ekr.20171213163216.1: *4* sd.format
def format(self, node, strip=True):
    
    class Formatter(leoAst.AstFormatter):
        level = 0
    
    s = Formatter().visit(node)
    line1 = g.splitLines(s)[0]
    line1 = line1.strip() if strip else line1.rstrip()
    return g.truncate(line1, 80)
#@+node:ekr.20171213155537.3: *4* sd.run
def run(self, files, dump=False, show_results=True):
    """Process all files"""
    t1 = time.time()
    for fn in files:
        s, e = g.readFileIntoString(fn)
        if s:
            print('=====', g.shortFileName(fn))
            s1 = g.toEncodedString(s)
            root = ast.parse(s1, filename='before', mode='exec')
            if dump:
                self.dump(fn, root)
            else:
                self.analyze(fn, root)
        else:
            g.trace('skipped', g.shortFileName(fn))
    t2 = time.time()
    if show_results:
        self.show_results()
    g.trace('done: %s files in %4.1f sec.' % (len(files), (t2 - t1)))
#@+node:ekr.20171213155537.7: *4* sd.show_results
def show_results(self):
    """Print a summary of the test results."""
    table = (
        ('assignments', self.assigns_d),
        ('calls', self.calls_d),
        ('classes', self.classes_d),
        ('defs', self.defs_d),
        ('returns', self.returns_d),
    )
    for name, d in table:
        print(f"{name}...")
        g.printDict({key: sorted(set(d.get(key))) for key in d})
#@+node:ekr.20171213174732.1: *4* sd.visit
def visit(self, node, types):
    if isinstance(node, types):
        yield self.format(node)
#@+node:ekr.20150604164113.1: *3* class ShowData
class ShowData:
    """The driver class for analysis project."""
    @others
#@+node:ekr.20150604165500.1: *4*  ctor
def __init__(self, c):
    """Ctor for ShowData controller class."""
    self.c = c
    self.files = None
    # Data.
    self.assigns_d = {}
    self.calls_d = {}
    self.classes_d = {}
    self.context_stack = []
    self.defs_d = {}
    self.returns_d = {}
    # Statistics
    self.n_matches = 0
    self.n_undefined_calls = 0
    self.tot_lines = 0
    self.tot_s = 0
#@+node:ekr.20150604163903.1: *4* run & helpers
def run(self, files):
    """Process all files"""
    self.files = files
    t1 = time.time()
    for fn in files:
        s, e = g.readFileIntoString(fn)
        if s:
            self.tot_s += len(s)
            g.trace('%8s %s' % ("{:,}".format(len(s)), g.shortFileName(fn)))
                # Print len(s), with commas.
            # Fast, accurate:
            # 1.9 sec for parsing.
            # 2.5 sec for Null AstFullTraverer traversal.
            # 2.7 sec to generate all strings.
            # 3.8 sec to generate all reports.
            s1 = g.toEncodedString(s)
            self.tot_lines += len(g.splitLines(s))
                # Adds less than 0.1 sec.
            node = ast.parse(s1, filename='before', mode='exec')
            ShowDataTraverser(self, fn).visit(node)
            # elif 0: # Too slow, too clumsy: 3.3 sec for tokenizing
                # readlines = g.ReadLinesClass(s).next
                # for token5tuple in tokenize.generate_tokens(readlines):
                    # pass
            # else: # Inaccurate. 2.2 sec to generate all reports.
                # self.scan(fn, s)
        else:
            g.trace('skipped', g.shortFileName(fn))
    t2 = time.time()
        # Get the time exlusive of print time.
    self.show_results()
    g.trace('done: %4.1f sec.' % (t2 - t1))
#@+node:ekr.20150605054921.1: *4* scan & helpers (a prototype: no longer used)
if 0:
    # The excellent prototype code, fast, easy but inaccurate.
    # It was a roadmap for the ShowDataTraverser class.

    # Regex patterns (were defined in the ctor)
    r_class = r'class[ \t]+([a-z_A-Z][a-z_A-Z0-9]*).*:'
    r_def = r'def[ \t]+([a-z_A-Z][a-z_A-Z0-9]*)[ \t]*\((.*)\)'
    r_return = r'(return[ \t].*)$'
    r_call = r'([a-z_A-Z][a-z_A-Z0-9]*)[ \t]*\(([^)]*)\)'
    r_all = re.compile(r'|'.join([r_class, r_def, r_return, r_call,]))

    def scan(self, fn, s):
        lines = g.splitLines(s)
        self.tot_lines += len(lines)
        for i, s in enumerate(lines):
            m = re.search(self.r_all, s)
            if m and not s.startswith('@'):
                self.match(fn, i, m, s)
#@+node:ekr.20150605063318.1: *5* match
def match(self, fn, i, m, s):
    """Handle the next match."""
    self.n_matches += 1
    indent = g.skip_ws(s, 0)
    # Update the context and enter data.
    if g.match_word(s, indent, 'def'):
        self.update_context(fn, indent, 'def', s)
        for i, name in enumerate(m.groups()):
            if name:
                aList = self.defs_d.get(name, [])
                def_tuple = self.context_stack[: -1], s
                aList.append(def_tuple)
                self.defs_d[name] = aList
                break
    elif g.match_word(s, indent, 'class'):
        self.update_context(fn, indent, 'class', s)
        for i, name in enumerate(m.groups()):
            if name:
                aList = self.classes_d.get(name, [])
                class_tuple = self.context_stack[: -1], s
                aList.append(class_tuple)
                self.classes_d[name] = aList
    elif s.find('return') > -1:
        context, name = self.context_names()
        j = s.find('#')
        if j > -1: s = s[: j]
        s = s.strip()
        if s:
            aList = self.returns_d.get(name, [])
            return_tuple = context, s
            aList.append(return_tuple)
            self.returns_d[name] = aList
    else:
        # A call.
        for i, name in enumerate(m.groups()):
            if name:
                context2, context1 = self.context_names()
                j = s.find('#')
                if j > -1:
                    s = s[: j]
                s = s.strip().strip(',').strip()
                if s:
                    aList = self.calls_d.get(name, [])
                    call_tuple = context2, context1, s
                    aList.append(call_tuple)
                    self.calls_d[name] = aList
                break
#@+node:ekr.20150605074749.1: *5* update_context
def update_context(self, fn, indent, kind, s):
    """Update context info when a class or def is seen."""
    while self.context_stack:
        fn2, kind2, indent2, s2 = self.context_stack[-1]
        if indent <= indent2:
            self.context_stack.pop()
        else:
            break
    context_tuple = fn, kind, indent, s
    self.context_stack.append(context_tuple)
    self.context_indent = indent
#@+node:ekr.20150604164546.1: *4* show_results & helpers
def show_results(self):
    """Print a summary of the test results."""
    make = True
    multiple_only = False # True only show defs defined in more than one place.
    c = self.c
    result = ['@killcolor\n']
    for name in sorted(self.defs_d):
        aList = self.defs_d.get(name, [])
        if len(aList) > 1 or not multiple_only: # not name.startswith('__') and (
            self.show_defs(name, result)
            self.show_calls(name, result)
            self.show_returns(name, result)
    self.show_undefined_calls(result)
    # Put the result in a new node.
    format = (
        'files: %s lines: %s chars: %s classes: %s\n'
        'defs: %s calls: %s undefined calls: %s returns: %s'
    )
    summary = format % (
        # g.plural(self.files),
        len(self.files),
        "{:,}".format(self.tot_lines),
        "{:,}".format(self.tot_s),
        "{:,}".format(len(self.classes_d.keys())),
        "{:,}".format(len(self.defs_d.keys())),
        "{:,}".format(len(self.calls_d.keys())),
        "{:,}".format(self.n_undefined_calls),
        "{:,}".format(len(self.returns_d.keys())),
    )
    result.insert(1, summary)
    result.extend(['', summary])
    if c and make:
        last = c.lastTopLevel()
        p2 = last.insertAfter()
        p2.h = 'global signatures'
        p2.b = '\n'.join(result)
        c.redraw(p=p2)
    print(summary)
#@+node:ekr.20150605160218.1: *5* show_calls
def show_calls(self, name, result):
    aList = self.calls_d.get(name, [])
    if not aList:
        return
    result.extend(['', f"    {len(aList)} call{g.plural(aList)}..."])
    w = 0
    calls = sorted(set(aList))
    for call_tuple in calls:
        context2, context1, s = call_tuple
        w = max(w, len(context2 or '') + len(context1 or ''))
    for call_tuple in calls:
        context2, context1, s = call_tuple
        pad = w - (len(context2 or '') + len(context1 or ''))
        if context2:
            result.append('%s%s::%s: %s' % (
                ' ' * (8 + pad), context2, context1, s))
        else:
            result.append('%s%s: %s' % (
                ' ' * (10 + pad), context1, s))
#@+node:ekr.20150605155601.1: *5* show_defs
def show_defs(self, name, result):
    aList = self.defs_d.get(name, [])
    name_added = False
    w = 0
    # Calculate the width
    for def_tuple in aList:
        context_stack, s = def_tuple
        if context_stack:
            fn, kind, context_s = context_stack[-1]
            w = max(w, len(context_s))
    for def_tuple in aList:
        context_stack, s = def_tuple
        if not name_added:
            name_added = True
            result.append('\n%s' % name)
            result.append(f"    {len(aList)} definition{g.plural(aList)}...")
        if context_stack:
            fn, kind, context_s = context_stack[-1]
            def_s = s.strip()
            pad = w - len(context_s)
            result.append('%s%s: %s' % (' ' * (8 + pad), context_s, def_s))
        else:
            result.append('%s%s' % (' ' * 4, s.strip()))
#@+node:ekr.20150605160341.1: *5* show_returns
def show_returns(self, name, result):
    aList = self.returns_d.get(name, [])
    if not aList:
        return
    result.extend(['', f"    {len(aList)} return{g.plural(aList)}..."])
    w, returns = 0, sorted(set(aList))
    for returns_tuple in returns:
        context, s = returns_tuple
        w = max(w, len(context or ''))
    for returns_tuple in returns:
        context, s = returns_tuple
        pad = w - len(context)
        result.append('%s%s: %s' % (' ' * (8 + pad), context, s))
#@+node:ekr.20150606092147.1: *5* show_undefined_calls
def show_undefined_calls(self, result):
    """Show all calls to undefined functions."""
    call_tuples = []
    for s in self.calls_d:
        i = 0
        while True:
            progress = i
            j = s.find('.', i)
            if j == -1:
                name = s[i:].strip()
                call_tuple = name, s
                call_tuples.append(call_tuple)
                break
            else:
                i = j + 1
            assert progress < i
    undef = []
    for call_tuple in call_tuples:
        name, s = call_tuple
        if name not in self.defs_d:
            undef.append(call_tuple)
    undef = list(set(undef))
    result.extend(['', f"{len(undef)} undefined call{g.plural(undef)}..."])
    self.n_undefined_calls = len(undef)
    # Merge all the calls for name.
    # There may be several with different s values.
    results_d = {}
    for undef_tuple in undef:
        name, s = undef_tuple
        calls = self.calls_d.get(s, [])
        aList = results_d.get(name, [])
        for call_tuple in calls:
            aList.append(call_tuple)
        results_d[name] = aList
    # Print the final results.
    for name in sorted(results_d):
        calls = results_d.get(name)
        result.extend(['', f"{name} {len(calls)} call{g.plural(calls)}..."])
        w = 0
        for call_tuple in calls:
            context2, context1, s = call_tuple
            if context2:
                w = max(w, 2 + len(context2) + len(context1))
            else:
                w = max(w, len(context1))
        for call_tuple in calls:
            context2, context1, s = call_tuple
            pad = w - (len(context2) + len(context1))
            if context2:
                result.append('%s%s::%s: %s' % (
                    ' ' * (2 + pad), context2, context1, s))
            else:
                result.append('%s%s: %s' % (
                    ' ' * (2 + pad), context1, s))
#@+node:ekr.20150605140911.1: *4* context_names
def context_names(self):
    """Return the present context name."""
    if self.context_stack:
        result = []
        for stack_i in -1, -2:
            try:
                fn, kind, indent, s = self.context_stack[stack_i]
            except IndexError:
                result.append('')
                break
            s = s.strip()
            assert kind in ('class', 'def'), kind
            i = g.skip_ws(s, 0)
            i += len(kind)
            i = g.skip_ws(s, i)
            j = g.skip_c_id(s, i)
            result.append(s[i: j])
        return reversed(result)
    return ['', '']
#@+node:ekr.20150606024455.1: *3* class ShowDataTraverser (AstFullTraverser)
class ShowDataTraverser(leoAst.AstFullTraverser):
    """
    Add data about classes, defs, returns and calls to controller's
    dictionaries.
    """

    def __init__(self, controller, fn):
        """Ctor for ShopDataTraverser class."""
        super().__init__()
        module_tuple = g.shortFileName(fn), 'module', g.shortFileName(fn)
            # fn, kind, s.
        self.context_stack = [module_tuple]
        self.controller = controller
        self.fn = g.shortFileName(fn)
        self.formatter = leoAst.AstFormatter()
            # leoAst.AstPatternFormatter()
    @others
#@+node:ekr.20150609053332.1: *4* sd.Helpers
#@+node:ekr.20150606035006.1: *5* sd.context_names
def context_names(self):
    """Return the present context names."""
    result = []
    n = len(self.context_stack)
    for i in n - 1, n - 2:
        if i >= 0:
            fn, kind, s = self.context_stack[i]
            assert kind in ('class', 'def', 'module'), kind
            if kind == 'module':
                result.append(s.strip())
            else:
                # Append the name following the class or def.
                i = g.skip_ws(s, 0)
                i += len(kind)
                i = g.skip_ws(s, i)
                j = g.skip_c_id(s, i)
                result.append(s[i: j])
        else:
            result.append('')
            break
    return reversed(result)
#@+node:ekr.20150609053010.1: *5* sd.format
def format(self, node, level, *args, **kwargs):
    """Return the formatted version of an Ast Node."""
    return self.formatter.format(node, level, *args, **kwargs).strip()
#@+node:ekr.20150606024455.62: *4* sd.visit
def visit(self, node):
    """
    Visit a *single* ast node. Visitors must visit their children
    explicitly.
    """
    method = getattr(self, 'do_' + node.__class__.__name__)
    method(node)

def visit_children(self, node):
    """Override to ensure this method is never called."""
    assert False, 'must visit children explicitly'
#@+node:ekr.20150609052952.1: *4* sd.Visitors
#@+node:ekr.20150607200422.1: *5* sd.Assign
def do_Assign(self, node):
    """Handle an assignment statement: Assign(expr* targets, expr value)"""
    value = self.format(self.visit(node.value), self.level)
    assign_tuples = []
    for target in node.targets:
        target = self.format(self.visit(target), self.level)
        s = '%s=%s' % (target, value)
        context2, context1 = self.context_names()
        assign_tuple = context2, context1, s
        assign_tuples.append(assign_tuple)
        aList = self.controller.assigns_d.get(target, [])
        aList.extend(assign_tuples)
        self.controller.calls_d[target] = aList
#@+node:ekr.20150607200439.1: *5* sd.AugAssign
def do_AugAssign(self, node):
    """
    Handle an augmented assignement:
    AugAssign(expr target, operator op, expr value).
    """
    target = self.format(self.visit(node.target), self.level)
    s = '%s=%s' % (target, self.format(self.visit(node.value), self.level))
    context2, context1 = self.context_names()
    assign_tuple = context2, context1, s
    aList = self.controller.assigns_d.get(target, [])
    aList.append(assign_tuple)
    self.controller.calls_d[target] = aList
#@+node:ekr.20150606024455.16: *5* sd.Call
def do_Call(self, node):
    """
    Handle a call statement:
    Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)
    """
    # Update data.
    s = self.format(node, self.level)
    name = self.format(node.func, self.level)
    context2, context1 = self.context_names()
    call_tuple = context2, context1, s
    aList = self.controller.calls_d.get(name, [])
    aList.append(call_tuple)
    self.controller.calls_d[name] = aList
    # Visit.
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
#@+node:ekr.20150606024455.3: *5* sd.ClassDef
def do_ClassDef(self, node):
    """
    Handle a class defintion:
    ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
    """
    # pylint: disable=arguments-differ
    # Format.
    if node.bases:
        bases = [self.format(z, self.level) for z in node.bases]
        s = 'class %s(%s):' % (node.name, ','.join(bases))
    else:
        s = 'class %s:' % node.name
    # Enter the new context.
    context_tuple = self.fn, 'class', s
    self.context_stack.append(context_tuple)
    # Update data.
    class_tuple = self.context_stack[: -1], s
    aList = self.controller.classes_d.get(node.name, [])
    aList.append(class_tuple)
    self.controller.classes_d[node.name] = aList
    # Visit.
    for z in node.bases:
        self.visit(z)
    for z in node.body:
        self.visit(z)
    for z in node.decorator_list:
        self.visit(z)
    # Leave the context.
    self.context_stack.pop()
#@+node:ekr.20150606024455.4: *5* sd.FunctionDef
def do_FunctionDef(self, node):
    """
    Visit a function defintion:
    FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
    """
    # pylint: disable=arguments-differ
    # Format.
    args = self.format(node.args, self.level) if node.args else ''
    s = 'def %s(%s):' % (node.name, args)
    # Enter the new context.
    context_tuple = self.fn, 'def', s
    self.context_stack.append(context_tuple)
    # Update data.
    def_tuple = self.context_stack[: -1], s
    aList = self.controller.defs_d.get(node.name, [])
    aList.append(def_tuple)
    self.controller.defs_d[node.name] = aList
    # Visit.
    for z in node.decorator_list:
        self.visit(z)
    self.visit(node.args)
    for z in node.body:
        self.visit(z)
    # Leave the context.
    self.context_stack.pop()
#@+node:ekr.20150606024455.55: *5* sd.Return
def do_Return(self, node):
    """Handle a 'return' statement: Return(expr? value)"""
    # Update data.
    s = self.format(node, self.level)
    context, name = self.context_names()
    aList = self.controller.returns_d.get(name, [])
    return_tuple = context, s
    aList.append(return_tuple)
    self.controller.returns_d[name] = aList
    # Visit.
    if node.value:
        self.visit(node.value)
#@+node:ekr.20171211163833.1: *3* class Stats
class Stats:
    """
    A basic statistics class.  Use this way:
        
        stats = Stats()
        stats.classes += 1
        stats.defs += 1
        stats.report()
    """

    d = {}
    
    def __getattr__(self, name):
        return self.d.get(name, 0)
        
    def __setattr__(self, name, val):
        self.d[name] = val
        
    def report(self):
        if self.d:
            n = max([len(key) for key in self.d])
            for key, val in sorted(self.d.items()):
                print('%*s: %s' % (n, key, val))
        else:
            print('no stats')
#@+node:ekr.20171211061816.1: *3* top-level test functions
#@+node:ekr.20150704135836.1: *4* testShowData (leoCheck.py)
def test(c, files):
    r"""
    A stand-alone version of @button show-data.  Call as follows:

        import leo.core.leoCheck as leoCheck
        files = (
            [
                # r'c:\leo.repo\leo-editor\leo\core\leoNodes.py',
            ] or
            leoCheck.ProjectUtils().project_files('leo')
        )
        leoCheck.test(files)
    """
    # pylint: disable=import-self
    import leo.core.leoCheck as leoCheck
    leoCheck.ShowData(c=c).run(files)
#@+node:ekr.20171211055756.1: *3* checkConventions (checkerCommands.py)
@g.command('check-conventions')
@g.command('cc')
def checkConventions(event):
    """Experimental script to test Leo's convensions."""
    c = event.get('c')
    if c:
        if c.changed: c.save()
        import importlib
        import leo.core.leoCheck as leoCheck
        importlib.reload(leoCheck)
        leoCheck.ConventionChecker(c).check()
#@+node:ekr.20200306045302.1: ** From leoImport.py
#@+node:ekr.20031218072017.3220: *3* ic.importFlattenedOutline (no longer used)
def importFlattenedOutline(self, files):  # Not a command, so no event arg.
    c = self.c; u = c.undoer
    if not c.p: return
    if not files: return
    self.setEncoding()
    fileName = files[0]  # files contains at most one file.
    g.setGlobalOpenDir(fileName)
    s, e = g.readFileIntoString(fileName)
    if s is None or not s.strip():
        return
    s = s.replace('\r', '')  # Fixes bug 626101.
    array = s.split("\n")
    # Convert the string to an outline and insert it after the current node.
    undoData = u.beforeInsertNode(c.p)
    # MORE files are more restrictive than tab-delimited outlines, so try them first.
    p = None
    c.endEditing()
    ###
        # importer = MORE_Importer(c)
        # if importer.check(s):
            # p = importer.import_lines(array, c.p)
    if not p:
        # Try to import a tab-delimited outline.
        importer = TabImporter(c)
        if importer.check(s, warn=False):
            p = importer.scan(s, fn=fileName, root=c.p)
    if p:
        c.validateOutline()
        p.setDirty()
        c.setChanged()
        u.afterInsertNode(p, 'Import', undoData)
        c.redraw(p)
    # elif not g.unitTesting:
        # g.es_print("not a valid MORE file", fileName)
#@+node:ekr.20200219073706.1: ** From leoTest.py
#@+node:ekr.20120220070422.10420: *3* Top-level functions (leoTest)
#@+node:ekr.20051104075904.97: *4* leoTest.py: factorial (a test of doctests)
# Some of these will fail now for Python 2.x.

def factorial(n):
    """Return the factorial of n, an exact integer >= 0.

    If the result is small enough to fit in an int, return an int.
    Else return a long.

    >>> [factorial(n) for n in range(6)]
    [1, 1, 2, 6, 24, 120]
    >>> factorial(30)
    265252859812191058636308480000000
    >>> factorial(-1)
    Traceback (most recent call last):
        ...
    ValueError: n must be >= 0

    Factorials of floats are OK, but the float must be an exact integer:
    >>> factorial(30.1)
    Traceback (most recent call last):
        ...
    ValueError: n must be exact integer
    >>> factorial(30.0)
    265252859812191058636308480000000

    It must also not be ridiculously large:
    >>> factorial(1e100)
    Traceback (most recent call last):
        ...
    OverflowError: n too large
    """
    import math
    if not n >= 0:
        raise ValueError("n must be >= 0")
    if math.floor(n) != n:
        raise ValueError("n must be exact integer")
    if n + 1 == n:  # catch a value like 1e300
        raise OverflowError("n too large")
    result = 1
    factor = 2
    while factor <= n:
        try:
            result *= factor
        except OverflowError:
            result = int(factor)
        factor += 1
    return result
#@+node:ekr.20200518051925.1: ** From leoKeys.py
#@+node:ekr.20171124074112.1: *3* c.executeAnyCommand (no longer used)
# def executeAnyCommand(self, command, event):
    # """Execute a command, no matter how defined."""
    # try:
        # return command(event)
    # except Exception:
        # g.es_exception()
        # return None
#@+node:ekr.20051106040126: *3* c.executeMinibufferCommand (no longer used)
def executeMinibufferCommand(self, commandName):
    c = self; k = c.k
    func = c.commandsDict.get(commandName)
    if func:
        event = g.app.gui.create_key_event(c)
        return k.masterCommand(commandName=None, event=event, func=func)
    g.error(f"no such command: {commandName} {g.callers()}")
    return None
#@+node:ekr.20170324143353.1: *3* k.commandExists (no longer used)
def commandExists(self, commandName):
    """
    Return the command handler for the given command name, or None.
    """
    ### This is supposed to allow for stuff after the command name.
    c, k = self.c, self
    commandName = commandName.strip()
    if not commandName:
        return None
    aList = commandName.split(None)
    if len(aList) == 1:
        k.givenArgs = []
    else:
        commandName = aList[0]
        k.givenArgs = aList[1:]
    func = c.commandsDict.get(commandName)
    return func
#@+node:ekr.20200523081446.1: *3* k.doKeyOnlyTasks (new: no longer used)
def doKeyOnlyTasks(self, event):
    """
    Do keystroke-related tasks related to commands.
    
    Return True if we should ignore the event.
    """
    ### c, k = self.c, self
    k = self
    assert event, g.callers()
    ###if not event: g.trace('Can not happen: no event')
        
    ### ch, stroke = event.char, event.stroke
    assert event.stroke, g.callers()
    ###
        # if not event.stroke:
            # g.trace('Can not happen: no stroke')
            # return True
    ### # Ignore all special keys.
    assert not k.isSpecialKey(event), g.callers()
    ###
        # if k.isSpecialKey(event):
            # g.trace('\n===== Can not happen: k.isSpecial')
            # return True
    ###
        # Remember the key.
        # k.setLossage(ch, stroke)
    ### # Handle keyboard-quit.
    if k.abortAllModesKey and event.stroke == k.abortAllModesKey:
        g.trace('\n===== Can not happen: Ctrl-G')
        ### k.keyboardQuit()
        ### return True
        assert False, g.callers()
    ###
        # # Ignore abbreviations.
        # if k.abbrevOn and c.abbrevCommands.expandAbbrev(event, stroke):
            # return False
    return False
#@+node:ekr.20180418025702.1: *3* k.doUnboundPlainKey & helper (no longer used)
def doUnboundPlainKey(self, event):
    """
    Handle unbound plain keys.
    Return True if k.masterKeyHandler should return.
    """
    c, k = self.c, self
    stroke, w = event.stroke, event.widget
    #
    # Ignore non-plain keys.
    if not k.isPlainKey(stroke):
        return False
    #
    # Ignore any keys in the background tree widget.
    if c.widget_name(w).startswith('canvas'):
        return False
    #
    # Ignore the char if it is bound to the auto-complete command.
    if self.isAutoCompleteChar(stroke):
        return False
    #
    # Handle the unbound key.
    g.trace(event.stroke)
    k.handleUnboundKeys(event)
    return True
#@+node:ekr.20110209083917.16004: *4* k.isAutoCompleteChar
def isAutoCompleteChar(self, stroke):
    """
    Return True if stroke is bound to the auto-complete in
    the insert or overwrite state.
    """
    k = self; state = k.unboundKeyAction
    assert g.isStrokeOrNone(stroke)
    if stroke and state in ('insert', 'overwrite'):
        for key in (state, 'body', 'log', 'text', 'all'):
            d = k.masterBindingsDict.get(key, {})
            if d:
                bi = d.get(stroke)
                if bi:
                    assert bi.stroke == stroke, f"bi: {bi} stroke: {stroke}"
                    if bi.commandName == 'auto-complete':
                        return True
    return False
#@+node:ekr.20061031131434.113: *3* k.endCommand (no longer used)
def endCommand(self, commandName):
    """Make sure Leo updates the widget following a command.

    Never changes the minibuffer label: individual commands must do that.
    """
    k = self; c = k.c
    # The command may have closed the window.
    if g.app.quitting or not c.exists: return
    # Set the best possible undoType: prefer explicit commandName to k.commandName.
    commandName = commandName or k.commandName or ''
    k.commandName = k.commandName or commandName or ''
    if commandName:
        if not k.inState():
            k.commandName = None
        Do *not* call this by default.  It interferes with undo.
            # c.frame.body.onBodyChanged(undoType='Typing')
        
            # if k.newMinibufferWidget:
                # c.widgetWantsFocusNow(k.newMinibufferWidget)
                # # g.pr('endCommand', g.app.gui.widget_name(k.newMinibufferWidget),g.callers())
                # k.newMinibufferWidget = None
#@+node:ekr.20061031131434.110: *3* k.handleDefaultChar (no longer used)
def handleDefaultChar(self, event, stroke):
    """
    Handle an unbound key, based on the event's widget.
    Do not assume that stroke exists.
    """
    c, k, w = self.c, self, event.widget
    name = c.widget_name(w)
    #
    # Ignore unbound alt-ctrl key
    if stroke and stroke.isAltCtrl() and k.ignore_unbound_non_ascii_keys:
        g.app.unitTestDict['handleUnboundChar-ignore-alt-or-ctrl'] = True
        return
    #
    # Handle events in the body pane.
    if name.startswith('body'):
        action = k.unboundKeyAction
        if action in ('insert', 'overwrite'):
            c.editCommands.selfInsertCommand(event, action=action)
        else:
            pass  # Ignore the key
        return
    #
    # Handle events in headlines.
    if name.startswith('head'):
        c.frame.tree.onHeadlineKey(event)
        return
    #
    # Handle events in the background tree.
    if name.startswith('canvas'):
        if not stroke:  # Not exactly right, but it seems to be good enough.
            c.onCanvasKey(event)
        return
    #
    # Handle events in the log pane.
    if name.startswith('log'):
        # Make sure we can insert into w.
        log_w = event.widget
        if not hasattr(log_w, 'supportsHighLevelInterface'):
            return
        # Send the event to the text widget, not the LeoLog instance.
        if not stroke:
            stroke = event.stroke
        if stroke:
            i = log_w.getInsertPoint()
            s = stroke.toGuiChar()
            log_w.insert(i, s)
        return
    #
    # Ignore all other events.
#@+node:ekr.20080510095819.1: *3* k.handleUnboundKeys (no longer used)
def handleUnboundKeys(self, event):
    """
    The last step of k.masterKeyHandler.
    Handle key events when no binding exists.
    """
    c, k = self.c, self
    stroke = event.stroke
    if not stroke:
        return
    # #1448: Very late special case for getArg state.
    #        This is not needed for other states.
    if stroke.isNumPadKey() and k.state.kind == 'getArg':
        stroke.removeNumPadModifier()
        k.getArg(event, stroke=stroke)
        return
    # Handle all unbound characters in command mode.
    if k.unboundKeyAction == 'command':
        w = g.app.gui.get_focus(c)
        if w and g.app.gui.widget_name(w).lower().startswith('canvas'):
            c.onCanvasKey(event)
        return
    # Ignore unbound F-keys.
    if stroke.isFKey():
        return
    # Handle a normal character in insert/overwrite.
    # <Return> is *not* a normal character.
    if (
        stroke and k.isPlainKey(stroke) and
        k.unboundKeyAction in ('insert', 'overwrite')
    ):
        c.insertCharFromEvent(event)
        return
    # Ignore unbound Alt/Ctrl keys.
    if stroke.isAltCtrl() and not self.enable_alt_ctrl_bindings:
        return
    # #868
    if stroke.isPlainNumPad():
        stroke.removeNumPadModifier()
        event.stroke = stroke
        c.insertCharFromEvent(event)
        return
    # #868
    if stroke.isNumPadKey():
        return  # To have effect, these must be bound.
    # Ignore unbound non-ascii character.
    if k.ignore_unbound_non_ascii_keys and not stroke.isPlainKey():
        return
    # Never insert escape or insert characters.
    if 'Escape' in stroke or 'Insert' in stroke:
        return
    # Handle the unbound character.
    c.insertCharFromEvent(event)
#@+node:ekr.20061031131434.105: *3* k.masterCommand (no longer used)
def masterCommand(self, commandName=None, event=None, func=None, stroke=None):
    """
    This is the central dispatching method.
    All commands and keystrokes pass through here.
    
    Return the value returned by the command, or None if no command is executed.
    """
    c, k = self.c, self
    if event: c.check_event(event)
    c.setLog()
    k.stroke = stroke  # Set this global for general use.
    ch = event.char if event else ''
    #
    # Ignore all special keys.
    if k.isSpecialKey(event):
        return None
    #
    # Compute func if not given.
    # It is *not* an error for func to be None.
    if commandName and not func:
        func = c.commandsDict.get(commandName.replace('&', ''))
        if not func:
            g.es_print(f"no command for @item {commandName!r}", color='red')
            return None
    commandName = commandName or func and func.__name__ or '<no function>'
    if 'keys' in g.app.debug:
        # A very important trace.
        g.trace(commandName, 'stroke', stroke)
    #
    # Remember the key.
    k.setLossage(ch, stroke)
    #
    # Handle keyboard-quit.
    if k.abortAllModesKey and stroke == k.abortAllModesKey:
        k.keyboardQuit()
        return None
    #
    # Ignore abbreviations.
    if k.abbrevOn and c.abbrevCommands.expandAbbrev(event, stroke):
        return None
    #
    # Invoke the command, if given.
    if func:
        return_value = c.doCommand(func, commandName, event=event)
        if c.exists:
            c.frame.updateStatusLine()
        return return_value
    #
    # Ignore unbound keys in a state.
    if k.inState():
        return None
    #
    # Finally, call k.handleDefaultChar.
    k.handleDefaultChar(event, stroke)
    if c.exists:
        c.frame.updateStatusLine()
    return None
#@+node:ekr.20061031131434.200: *3* k.universalDispatcher & helpers (no longer used)
def universalDispatcher(self, event):
    """Handle accumulation of universal argument."""
    << about repeat counts >>
    k = self
    state = k.getState('u-arg')
    stroke = event.stroke if event else ''
    if state == 0:
        k.dispatchEvent = event
        # The call should set the label.
        k.setState('u-arg', 1, k.universalDispatcher)
        k.repeatCount = 1
    elif state == 1:
        char = event.char if event else ''
        if char == 'Escape':
            k.keyboardQuit()
        elif char == k.universalArgKey:
            k.repeatCount = k.repeatCount * 4
        elif char.isdigit() or char == '-':
            k.updateLabel(event)
        elif char in (
            'Alt_L', 'Alt_R',
            'Control_L', 'Control_R',
            'Meta_L', 'Meta_R',
            'Shift_L', 'Shift_R',
        ):
            k.updateLabel(event)
        else:
            # *Anything* other than C-u, '-' or a numeral is taken to be a command.
            val = k.getLabel(ignorePrompt=True)
            try: n = int(val) * k.repeatCount
            except ValueError: n = 1
            k.clearState()
            event = k.dispatchEvent
            k.executeNTimes(event, n)
            k.keyboardQuit()
    elif state == 2:
        k.doControlU(event, stroke)
#@+node:ekr.20061031131434.201: *4* << about repeat counts >>
@nocolor
@  Any Emacs command can be given a numeric argument. Some commands interpret the
argument as a repetition count. For example, giving an argument of ten to the
key C-f (the command forward-char, move forward one character) moves forward ten
characters. With these commands, no argument is equivalent to an argument of
one. Negative arguments are allowed. Often they tell a command to move or act
backwards.

If your keyboard has a META key, the easiest way to specify a numeric argument
is to type digits and/or a minus sign while holding down the the META key. For
example,

M-5 C-n

moves down five lines. The characters Meta-1, Meta-2, and so on, as well as
Meta--, do this because they are keys bound to commands (digit-argument and
negative-argument) that are defined to contribute to an argument for the next
command.

Another way of specifying an argument is to use the C-u (universal-argument)
command followed by the digits of the argument. With C-u, you can type the
argument digits without holding down shift keys. To type a negative argument,
start with a minus sign. Just a minus sign normally means -1. C-u works on all
terminals.

C-u followed by a character which is neither a digit nor a minus sign has the
special meaning of "multiply by four". It multiplies the argument for the next
command by four. C-u twice multiplies it by sixteen. Thus, C-u C-u C-f moves
forward sixteen characters. This is a good way to move forward "fast", since it
moves about 1/5 of a line in the usual size screen. Other useful combinations
are C-u C-n, C-u C-u C-n (move down a good fraction of a screen), C-u C-u C-o
(make "a lot" of blank lines), and C-u C-k (kill four lines).

Some commands care only about whether there is an argument and not about its
value. For example, the command M-q (fill-paragraph) with no argument fills
text; with an argument, it justifies the text as well. (See section Filling
Text, for more information on M-q.) Just C-u is a handy way of providing an
argument for such commands.

Some commands use the value of the argument as a repeat count, but do something
peculiar when there is no argument. For example, the command C-k (kill-line)
with argument n kills n lines, including their terminating newlines. But C-k
with no argument is special: it kills the text up to the next newline, or, if
point is right at the end of the line, it kills the newline itself. Thus, two
C-k commands with no arguments can kill a non-blank line, just like C-k with an
argument of one. (See section Deletion and Killing, for more information on
C-k.)

A few commands treat a plain C-u differently from an ordinary argument. A few
others may treat an argument of just a minus sign differently from an argument
of -1. These unusual cases will be described when they come up; they are always
to make the individual command more convenient to use.
#@+node:ekr.20061031131434.202: *4* k.executeNTimes
def executeNTimes(self, event, n):

    c, k = self.c, self
    w = event and event.widget
    stroke = event.stroke if event else ''
    if not stroke: return
    if stroke == k.fullCommandKey:
        for z in range(n):
            k.fullCommand(event)
    else:
        bi = k.getPaneBinding(stroke, event and event.widget)
        if bi:
            for z in range(n):
                event = g.app.gui.create_key_event(c, event=event, w=w)
                k.masterCommand(
                    commandName=None,
                    event=event,
                    func=bi.func,
                    stroke=stroke,
                )
        else:
            for z in range(n):
                k.masterKeyHandler(event)
#@+node:ekr.20061031131434.203: *4* doControlU
def doControlU(self, event, stroke):
    k = self
    ch = event.char if event else ''
    k.setLabelBlue(f"Control-u {g.stripBrackets(stroke)}")
    if ch == '(':
        k.clearState()
        k.resetLabel()
#@+node:ekr.20120208064440.10201: *3* k.NEWgeneralModeHandler (NEW MODES)
def NEWgeneralModeHandler(self, event,
    commandName=None,
    func=None,
    modeName=None,
    nextMode=None,
    prompt=None
):
    """Handle a mode defined by an @mode node in leoSettings.leo."""
    k = self; c = k.c
    state = k.getState(modeName)
    if state == 0:
        k.inputModeName = modeName
        k.modePrompt = prompt or modeName
        k.modeWidget = event and event.widget
        k.setState(modeName, 1, handler=k.generalModeHandler)
        self.initMode(event, modeName)
        # Careful: k.initMode can execute commands that will destroy a commander.
        if g.app.quitting or not c.exists: return
        if not k.silentMode:
            if c.config.getBool('showHelpWhenEnteringModes'):
                k.modeHelp(event)
            else:
                c.frame.log.hideTab('Mode')
    elif not func:
        g.trace('No func: improper key binding')
    else:
        if commandName == 'mode-help':
            func(event)
        else:
            self.endMode()
            # New in 4.4.1 b1: pass an event describing the original widget.
            if event:
                event.w = event.widget = k.modeWidget
            else:
                event = g.app.gui.create_key_event(c, w=k.modeWidget)
            func(event)
            if g.app.quitting or not c.exists:
                pass
            elif nextMode in (None, 'none'):
                # Do *not* clear k.inputModeName or the focus here.
                # func may have put us in *another* mode.
                pass
            elif nextMode == 'same':
                silent = k.silentMode
                k.setState(modeName, 1, handler=k.generalModeHandler)
                self.reinitMode(modeName)  # Re-enter this mode.
                k.silentMode = silent
            else:
                k.silentMode = False  # All silent modes must do --> set-silent-mode.
                self.initMode(event, nextMode)  # Enter another mode.
                # Careful: k.initMode can execute commands that will destroy a commander.
                # if g.app.quitting or not c.exists: return
#@+node:ekr.20200908101251.1: ** From leoApp.py
#@+node:ekr.20190826021428.1: *3* app.saveGlobalWindowState (no longer used)
def saveGlobalWindowState(self):
    """
    Save the window geometry and layout of dock widgets and toolbars
    for Leo's *global* QMainWindow.
    
    Called by g.app.finishQuit. 
    """
    return ###
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    main_window = getattr(g.app.gui, 'main_window', None)
    if not main_window:
        if trace:
            if hasattr(g.app.gui, 'main_window'):
                g.trace('g.app.gui.main_window is None')
            else:
                g.trace('no ivar: g.app.gui.main_window')
        return
    #
    # Save the state
    key = 'globalWindowState:'
    val = main_window.saveState()
        # Method is a QMainWindow method.
    try:
        val = bytes(val)  # PyQt4
    except Exception:
        val = bytes().join(val)  # PySide
    if trace: g.trace(f"set key: {key}:")
    g.app.db[key] = base64.encodebytes(val).decode('ascii')
#@+node:ekr.20190826022349.1: *3* app.restoreGlobalWindowState (no longer used)
def restoreGlobalWindowState(self):
    """
    Restore the layout of global dock widgets and toolbars.
    """
    #
    # Note for #1189: The windows has already been properly resized
    #                 by the time this method is called.
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    main_window = getattr(g.app.gui, 'main_window', None)
    if not main_window:
        if trace:
            if hasattr(g.app.gui, 'main_window'):
                g.trace('g.app.gui.main_window is None')
            else:
                g.trace('no ivar: g.app.gui.main_window')
        return
    g.trace('===== g.app.gui.main_window', main_window)
    return ###
    #
    # Support --init-docks.
    # #1196. Let Qt use it's own notion of a default layout.
    #        This should work regardless of the central widget.
    if g.app.init_docks:
        if trace: g.trace('--init-docks')
        return
    key = 'globalWindowState:'
    val = self.db.get(key)
    if val:
        if trace: g.trace(f"found key: {key}")
        try:
            val = base64.decodebytes(val.encode('ascii'))
                # Elegant pyzo code.
            main_window.restoreState(val)
            return
        except Exception as err:
            g.trace(f"bad value: {key} {err}")
            return
    # This is not an error.
    if trace: g.trace(f"missing key: {key}")
#@+node:ekr.20200915114300.1: ** ----- Deleted qt docks code
Rev 96f282672de5c11d00 in devel contains the old code.
#@+node:ekr.20200916074952.1: *3* deleted code snippets
#@+node:ekr.20200915153200.1: *4* from << LeoApp: global data >>
    self.defaultWindowState = b'\x00\x00\x00\xff\x00\x00\x00\x00\xfd\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x01\x19\x00\x00\x01^\xfc\x02\x00\x00\x00\x01\xfb\x00\x00\x00\x12\x00d\x00o\x00c\x00k\x00.\x00t\x00a\x00b\x00s\x01\x00\x00\x006\x00\x00\x01^\x00\x00\x00\x1b\x00\xff\xff\xff\x00\x00\x00\x03\x00\x00\x03\x1e\x00\x00\x00\xc3\xfc\x01\x00\x00\x00\x02\xfb\x00\x00\x00\x12\x00d\x00o\x00c\x00k\x00.\x00b\x00o\x00d\x00y\x01\x00\x00\x00\x00\x00\x00\x01\xfd\x00\x00\x001\x00\xff\xff\xff\xfb\x00\x00\x00\x16\x00d\x00o\x00c\x00k\x00.\x00R\x00e\x00n\x00d\x00e\x00r\x01\x00\x00\x02\x05\x00\x00\x01\x19\x00\x00\x001\x00\xff\xff\xff\x00\x00\x01\xfd\x00\x00\x01^\x00\x00\x00\x04\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x08\xfc\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x10\x00i\x00c\x00o\x00n\x00-\x00b\x00a\x00r\x01\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x01\x00\x00\x00$\x00m\x00i\x00n\x00i\x00b\x00u\x00f\x00f\x00e\x00r\x00-\x00t\x00o\x00o\x00l\x00b\x00a\x00r\x01\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
        #
        # For self.restoreWindowState the first time Leo is run.
        # Use the print-window-state to print this value after arranging the docks to your liking.
        # Important: the outline-pane *must* be the central widget.
#@+node:ekr.20200915153256.1: *4* from dw.addNewEditor
    #
    # Create dock, splitting the body dock.
    dock = self.addEditorDock()
    #
    # Create the editor
    widget = qt_text.LeoQTextBrowser(None, c, self)
    widget.setObjectName('richTextEdit')
    wrapper = qt_text.QTextEditWrapper(widget, name='body', c=c)
    self.packLabel(widget)
    #
    # Inject ivars, set bindings, etc.
    inner_frame = self.leo_body_inner_frame
        # Inject ivars *here*, regardless of docking.
    body.injectIvars(inner_frame, name, p, wrapper)
    body.updateInjectedIvars(widget, p)
    wrapper.setAllText(p.b)
    wrapper.see(0)
    c.k.completeAllBindingsForWidget(wrapper)
    if isinstance(widget, QtWidgets.QTextEdit):
        colorizer = leoColorizer.make_colorizer(c, widget, wrapper)
        colorizer.highlighter.setDocument(widget.document())
    else:
        # Scintilla only.
        body.recolorWidget(p, wrapper)
    return dock, wrapper
#@+node:ekr.20200915153424.1: *4* from dw.createBodyFrame
    #
    # Create widgets.
    #
    # bodyFrame has a VGridLayout.
    bodyFrame = self.createFrame(parent, 'bodyFrame')
    grid = self.createGrid(bodyFrame, 'bodyGrid')
    #
    # innerFrame has a VBoxLayout.
    innerFrame = self.createFrame(bodyFrame, 'innerBodyFrame')
    box = self.createVLayout(innerFrame, 'bodyVLayout', spacing=0)
    #
    # Pack the body alone or *within* a LeoLineTextWidget.
    body = self.createText(None, 'richTextEdit')  # A LeoQTextBrowser
    if self.use_gutter:
        lineWidget = qt_text.LeoLineTextWidget(c, body)
        box.addWidget(lineWidget)
    else:
        box.addWidget(body)
    grid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Official ivars
    self.richTextEdit = body
    self.leo_body_frame = bodyFrame
    self.leo_body_inner_frame = innerFrame
    return bodyFrame
#@+node:ekr.20200915153609.1: *4* from dw.setMainWindowOptions
    dw.setDockNestingEnabled(False)
    dw.setDockOptions(
        QtWidgets.QMainWindow.AllowTabbedDocks |
        QtWidgets.QMainWindow.AnimatedDocks)
#@+node:ekr.20200915153647.1: *4* from LeoQtBody.delete_editor_command
    #
    # Sanity checks.
    if not dock:
        dock = w.parent().parent()
        if (getattr(w, 'leo_name', None) == '1' or
            not isinstance(dock, QtWidgets.QDockWidget)
        ):
            g.warning('can not delete main editor')
            return
    #
    # Actually delete the dock.
    if dock in dw.added_editor_docks:
        dw.added_editor_docks.remove(dock)
    dw.removeDockWidget(dock)
        # A QMainWidget method.
    #
    # Select another editor.
    w.leo_label = None
    new_wrapper = list(d.values())[0]
    self.numberOfEditors -= 1
    self.selectEditor(new_wrapper)
#@+node:ekr.20200915153758.1: *4* from LeoQtBody.select/unselectLabel
        if isinstance(label, QtWidgets.QDockWidget):
            # #1517.
            label.setWindowTitle(c.p.h)
        el
#@+node:ekr.20200915153720.1: *4* from LeoQtLog.createTab
        if g.app.dock and c.config.getBool('dockable-log-tabs', default=False):
            # #1154: Support docks in the Log pane.
            dw = c.frame.top
            dock = g.app.gui.create_dock_widget(
                closeable=True, moveable=True, height=50, name=tabName)  # was 100: #1339.
                    # #1207: all plugins docks should be closeable.
            dock.setWidget(contents)
            area = QtCore.Qt.RightDockWidgetArea
            dw.addDockWidget(area, dock)
        else:
#@+node:ekr.20200916075026.1: *3* deleted functions
#@+node:ekr.20200303094809.1: *4*  function:dock_widget
def dock_widget(w):
    """Return the dock widget containing widget w."""
    while w:
        if isinstance(w, QtWidgets.QDockWidget):
            return w
        w = w.parent()
    return None
#@+node:ekr.20200303104851.1: *4* 'hide-body-dock'
@g.command('hide-body-dock')
def hideBodyDock(event):
    '''Hide the Body dock'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.body.widget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105224.1: *4* 'hide-outline-dock'
@g.command('hide-outline-dock')
def hideOutlineDock(event):
    '''Hide the Outline dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.tree.treeWidget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105106.1: *4* 'hide-tabs-dock'
@g.command('hide-tabs-dock')
def hideTabsDock(event):
    '''Hide the Tabs dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.log.logWidget
    dock = dock_widget(w)
    if dock:
        dock.hide()
#@+node:ekr.20200303105513.1: *4* 'show-body-dock'
@g.command('show-body-dock')
def showBodyDock(event):
    '''Show the Body dock'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.body.widget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20200303105712.1: *4* 'show-outline-dock'
@g.command('show-outline-dock')
def showOutlineDock(event):
    '''Show the Outline dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.tree.treeWidget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20200303105622.1: *4* 'show-tabs-dock'
@g.command('show-tabs-dock')
def showTabsDock(event):
    '''Show the Tabs dock.'''
    c = event.get('c')
    if not c:
        return
    w = c.frame.log.logWidget
    dock = dock_widget(w)
    if dock:
        dock.show()
#@+node:ekr.20190616092159.1: *4* app.get_central_widget
def get_central_widget(self, c):

    assert self.dock, g.callers()
    s = c.config.getString('central-dock-widget')
    if s:
        s = s.lower()
        if s in ('body', 'outline', 'tabs'):
            return s
    return 'outline'
#@+node:ekr.20200305102656.1: *4* app.restoreEditorDockState
def restoreEditorDockState(self, c):

    trace = any([z in g.app.debug for z in ('dock', 'select')]) and not g.app.unitTesting
    tag = 'app.restoreEditorDockState'
    body = c.frame.body
    dw = c.frame.top
    if not dw:
        return
    aps_s= c.db.get('added_editor_aps', '')
    dock_names_s = c.db.get('added_editor_docks', '')
    if not aps_s or not dock_names_s:
        return
    aps = aps_s.split(';')
    dock_names = dock_names_s.split(';')
    if len(aps) != len(dock_names):
        g.trace('can not happen')
        return
    if trace: g.trace('START')
    #
    # #1527: Part 1: Inject leo_wrapper ivar.
    #                Similar to LeoQtBody.injectIvars.
    wrapper = body.wrapper
    w = wrapper.widget
    w.leo_wrapper = wrapper
    if trace:
        print(f"{tag:>30}: {wrapper} <dock for main body>")
    #
    # #1527: Part 2: Pack the Body's label so it tracks p.h.
    #                Similar to code for the 'add-editor' command.
    if body.use_gutter:
        dw.packLabel(w.parent(), n=1)
        w.leo_label = w.parent().leo_label
    else:
        dw.packLabel(w, n=1)
    #
    # Restore all *added* editors.
    d = body.editorWrappers
    for i, dock_name in enumerate(dock_names):
        ap = aps[i]
        f, wrapper = dw.addNewEditor(dock_name)
        d[dock_name] = wrapper
        p = c.archivedPositionToPosition(ap)
        body.injectIvars(
            parentFrame = f,
            name = dock_name,
            p = p,
            wrapper = wrapper,
        )
        w =  c.frame.body.wrapper.widget
        body.updateInjectedIvars(w, p)
        body.selectLabel(wrapper)
        body.selectEditor(wrapper)
        if trace:
            print(f"{tag:>30}: {wrapper} {dock_name}")
    if trace: g.trace('END')
#@+node:ekr.20190528045549.1: *4* app.restoreWindowState
ekr_val = None

def restoreWindowState(self, c):
    """
    Restore the layout of dock widgets and toolbars, using the per-file
    state of the *first* loaded .leo file, or the global state.
    
    Note: The window's position or size has already been restored.
    """
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    tag = 'app.restoreWindowState:'
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    dw = c.frame.top
    if not dw or not hasattr(dw, 'restoreState'):
        if trace: g.trace('no dw.restoreState. dw:', repr(dw))
        return
    # First, restore the editor state.
    self.restoreEditorDockState(c)
    #
    # Support --init-docks.
    # #1196. Let Qt use it's own notion of a default layout.
    #        This should work regardless of the central widget.
    if g.app.init_docks:
        if trace: g.trace('using qt default layout')
        return
    sfn = c.shortFileName()
    table = (
        # First, try the per-outline state.
        (f"windowState:{c.fileName()}", dw.restoreState),
        # Restore the actual window state.
        ('windowState:', dw.restoreState),
    )
    for key, method in table:
        val = self.db.get(key)
        if val:
            if trace:
                g.trace(f"{sfn} found key: {key}")
            try:
                val = base64.decodebytes(val.encode('ascii'))
                    # Elegant pyzo code.
                method(val)
                return
            except Exception as err:
                g.trace(f"{sfn} bad value: {key} {err}")
        # This is not an error.
        elif trace:
            g.trace(f"{sfn} missing key: {key}")
    #
    # #1190 (bad initial layout)
    # Use a pre-defined layout (magic number).
    # The print-window-state prints this magic number for a *given* layout.
    # But this number will work *only* if the central widgets match.
    try:
        central_widget = self.get_central_widget(c)
        # central_widget = c.config.getString('central-dock-widget')
        # if central_widget:
            # central_widget = central_widget.lower()
        if central_widget in (None, 'outline'):
            if trace:
                print(tag, 'using app.defaultWindowState')
            dw.restoreState(self.defaultWindowState)
        elif trace:
            print(tag)
            print('central widget does not match default')
            print('using qt default window state')
        # See print-window-state.
    except Exception:
        g.es_print(tag, 'unexpected exception setting window state')
        g.es_exception()
#@+node:ekr.20190528045643.1: *4* app.saveWindowState
def saveWindowState(self, c):
    """
    Save the window geometry and layout of dock widgets and toolbars.
    
    This is called for all closed windows.
    """
    trace = any([z in g.app.debug for z in ('dock', 'cache', 'size', 'startup')])
    if not g.app.dock:
        if trace: g.trace('g.app.dock is False')
        return
    dw = c.frame.top
    if not dw or not hasattr(dw, 'saveState'):
        if trace: g.trace('no dw.saveState. dw:', repr(dw))
        return
    table = (
        # Save a default *global* state, for *all* outline files.
        ('windowState:', dw.saveState),
        # Save a per-file state.
        (f"windowState:{c.fileName()}", dw.saveState),
        # Do not save/restore window geometry. That is done elsewhere.
            # (f"windowGeometry:{c.fileName()}" , dw.saveGeometry),
    )
    for key, method in table:
        # This is pyzo code...
        val = method()
            # Method is a QMainWindow method.
        try:
            val = bytes(val)  # PyQt4
        except Exception:
            val = bytes().join(val)  # PySide
        val = base64.encodebytes(val).decode('ascii')
        if trace:
            g.trace(f"{c.shortFileName()} set key: {key}")         
        g.app.db[key] = val
#@+node:ekr.20190613062749.1: *4* debug.print-window-state
@cmd('print-window-state')
def printWindowState(self, event=None):
    """
    For Leo's core developers: print QMainWindow.saveState().
    
    This is the value to be assigned to g.app.defaultWindowState.
    
    Warning: this window state should *only* be used for new users! #1190.
    
    Recommended procedure:

    - Set @bool user-vr-dock = False.  Close Leo.
    - Clear .leo/db caches. Reopen Leo.
    - Make sure Render dock is visible to left of Body dock.
    - Set @bool user-vr-dock = True. Close Leo & reopen.
      The vr dock will be moveable.  Don't move it!
    - Do print-window-state.
    - Change g.app.defaultWindowState.
    """
    c = event.get('c')
    if c:
        print(c.frame.top.saveState())
        if c.config.getBool('dockable-log-tabs', default=False):
            print('Warning: @bool dockable-log-tabs is True')
        central_widget = g.app.get_central_widget(c)
        if central_widget != 'outline':
            print(f"Warning: @string central-dock-widget is {central_widget!r}")
    else:
        print('no c')
#@+node:ekr.20190523115826.1: *4* dw.addEditorDock
added_bodies = 0

def addEditorDock(self, closeable=True, moveable=True):
    """Add an editor dock"""
    #
    # Create the new dock.
    dw, c = self, self.leo_c
    dw.added_bodies += 1
    dock = g.app.gui.create_dock_widget(
        closeable=closeable,
        moveable=moveable,
        height=50,
        name=c.p.h,
    )
    dw.added_editor_docks.append(dock)
    w = dw.createBodyPane(parent=None)
    dock.setWidget(w)
    dw.splitDockWidget(dw.body_dock, dock, QtCore.Qt.Horizontal)
    #
    # monkey-patch dock.closeEvent

    def patched_closeEvent(event=None):
        c.frame.body.delete_editor_command(event, dock=dock)

    dock.closeEvent = patched_closeEvent
    return dock
#@+node:ekr.20190522165123.1: *4* dw.createAllDockWidgets
def createAllDockWidgets(self):
    """Create all the dock widgets."""
    c, dw = self.leo_c, self
    #
    # Compute constants.
    Qt = QtCore.Qt
    bottom, top = Qt.BottomDockWidgetArea, Qt.TopDockWidgetArea
    lt, rt = Qt.LeftDockWidgetArea, Qt.RightDockWidgetArea
    g.placate_pyflakes(bottom, lt, rt, top)
    #
    # Create all the docks.
    central_widget = g.app.get_central_widget(c)
    dockable = c.config.getBool('dockable-log-tabs', default=False)
    table = [
        (True, 50, lt, 'outline', dw.createOutlineDock),  # was 100: #1339.
        (True, 50, bottom, 'body', dw.createBodyPane),  # was 100: #1339.
        (True, 50, rt, 'tabs', dw.createTabsDock),  # was 20: #1339.
        (dockable, 20, rt, 'find', dw.createFindDockOrTab),
        (dockable, 20, rt, 'spell', dw.createSpellDockOrTab),
    ]
    for make_dock, height, area, name, creator in table:
        w = creator(parent=None)
        if not make_dock:
            setattr(dw, f"{name}_dock", None)
            continue
        dock = g.app.gui.create_dock_widget(
            closeable=name != central_widget,
            moveable=name != central_widget,
            height=0,
            name=name)
        dock.setWidget(w)
        # Remember the dock.
        setattr(dw, f"{name}_dock", dock)
        if name == central_widget:
            dw.setCentralWidget(dock)
                # Important: the central widget should be a dock.
            dock.show()  # #1327.
        else:
            dw.addDockWidget(area, dock)
    #
    # Create minibuffer.
    bottom_toolbar = QtWidgets.QToolBar(dw)
    bottom_toolbar.setObjectName('minibuffer-toolbar')
    bottom_toolbar.setWindowTitle('Minibuffer')
    dw.addToolBar(Qt.BottomToolBarArea, bottom_toolbar)
    w = dw.createMiniBuffer(bottom_toolbar)
    bottom_toolbar.addWidget(w)
    #
    # Create other widgets...
    dw.createMenuBar()
    dw.createStatusBar(dw)
#@+node:ekr.20190527120808.1: *4* dw.createFindDockOrTab
def createFindDockOrTab(self, parent):
    """Create a Find dock or tab in the Log pane."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create widgets.
    findTab = QtWidgets.QWidget()
    findTab.setObjectName('findTab')
    findScrollArea = QtWidgets.QScrollArea()
    findScrollArea.setObjectName('findScrollArea')
    #
    # For LeoFind.finishCreate.
    self.findScrollArea = findScrollArea
    self.findTab = findTab
    #
    # Create a tab in the log Dock, if necessary.
    if not c.config.getBool('dockable-log-tabs', default=False):
        self.tabWidget.addTab(findScrollArea, 'Find')
    return findScrollArea
#@+node:ekr.20190528112002.1: *4* dw.createOutlineDock
def createOutlineDock(self, parent):
    """Create the widgets and ivars for Leo's outline."""
    # Create widgets.
    treeFrame = self.createFrame(parent, 'outlineFrame',
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    innerFrame = self.createFrame(treeFrame, 'outlineInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred)
    treeWidget = self.createTreeWidget(innerFrame, 'treeWidget')
    grid = self.createGrid(treeFrame, 'outlineGrid')
    grid.addWidget(innerFrame, 0, 0, 1, 1)
    innerGrid = self.createGrid(innerFrame, 'outlineInnerGrid')
    innerGrid.addWidget(treeWidget, 0, 0, 1, 1)
    # Official ivars...
    self.treeWidget = treeWidget
    return treeFrame
#@+node:ekr.20190527120829.1: *4* dw.createSpellDockOrTab
def createSpellDockOrTab(self, parent):
    """Create a Spell dock  or tab in the Log pane."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create an outer widget.
    spellTab = QtWidgets.QWidget()
    spellTab.setObjectName('docked.spellTab')
    #
    # Create the contents.
    self.createSpellTab(spellTab)
    #
    # Create the Spell tab in the Log dock, if necessary.
    if not c.config.getBool('dockable-log-tabs', default=False):
        tabWidget = self.tabWidget
        tabWidget.addTab(spellTab, 'Spell')
        tabWidget.setCurrentIndex(1)
    return spellTab
#@+node:ekr.20190527163203.1: *4* dw.createTabbedLogDock
def createTabbedLogDock(self, parent):
    """Create a tabbed (legacy) Log dock."""
    assert g.app.dock
    assert not parent, repr(parent)
    c = self.leo_c
    #
    # Create the log frame.
    logFrame = self.createFrame(parent, 'logFrame',
        vPolicy=QtWidgets.QSizePolicy.Minimum)
    innerFrame = self.createFrame(logFrame, 'logInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred,
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    tabWidget = self.createTabWidget(innerFrame, 'logTabWidget')
    #
    # Pack.
    innerGrid = self.createGrid(innerFrame, 'logInnerGrid')
    innerGrid.addWidget(tabWidget, 0, 0, 1, 1)
    outerGrid = self.createGrid(logFrame, 'logGrid')
    outerGrid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Create the Find tab, embedded in a QScrollArea.
    findScrollArea = QtWidgets.QScrollArea()
    findScrollArea.setObjectName('findScrollArea')
    # Find tab.
    findTab = QtWidgets.QWidget()
    findTab.setObjectName('findTab')
    # Fix #516:
    use_minibuffer = c.config.getBool('minibuffer-find-mode', default=False)
    use_dialog = c.config.getBool('use-find-dialog', default=False)
    if not use_minibuffer and not use_dialog:
        tabWidget.addTab(findScrollArea, 'Find')
    # Do this later, in LeoFind.finishCreate
    self.findScrollArea = findScrollArea
    self.findTab = findTab
    #
    # Spell tab.
    spellTab = QtWidgets.QWidget()
    spellTab.setObjectName('spellTab')
    tabWidget.addTab(spellTab, 'Spell')
    self.createSpellTab(spellTab)
    tabWidget.setCurrentIndex(1)
    #
    # Official ivars
    self.tabWidget = tabWidget  # Used by LeoQtLog.
    return logFrame
#@+node:ekr.20190527121112.1: *4* dw.createTabsDock
def createTabsDock(self, parent):
    """Create the Tabs dock."""
    assert g.app.dock
    assert not parent, repr(parent)
    #
    # Create the log contents
    logFrame = self.createFrame(None, 'logFrame',
        vPolicy=QtWidgets.QSizePolicy.Minimum)
    innerFrame = self.createFrame(logFrame, 'logInnerFrame',
        hPolicy=QtWidgets.QSizePolicy.Preferred,
        vPolicy=QtWidgets.QSizePolicy.Expanding)
    tabWidget = self.createTabWidget(innerFrame, 'logTabWidget')
    #
    # Pack. This *is* required.
    innerGrid = self.createGrid(innerFrame, 'logInnerGrid')
    innerGrid.addWidget(tabWidget, 0, 0, 1, 1)
    outerGrid = self.createGrid(logFrame, 'logGrid')
    outerGrid.addWidget(innerFrame, 0, 0, 1, 1)
    #
    # Official ivars
    self.tabWidget = tabWidget  # Used by LeoQtLog.
    return logFrame
#@+node:ekr.20190724172314.1: *4* qt: show-hide-body-dock
@g.command('show-hide-body-dock')
def show_hide_body_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.body_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'body':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724172258.1: *4* qt: show-hide-outline-dock
@g.command('show-hide-outline-dock')
def show_hide_outline_dock(event):
    """Show or hide the Outline dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.outline_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'outline':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724172547.1: *4* qt: show-hide-render-dock
@g.command('show-hide-render-dock')
def show_hide_render_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    pc = g.app.pluginsController
    vr = pc.getPluginModule('leo.plugins.viewrendered')
    x = vr and vr.controllers.get(c.hash())
    dock = x and x.leo_dock
    if not dock:
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190724170436.1: *4* qt: show-hide-tabs-dock
@g.command('show-hide-tabs-dock')
def show_hide_tabs_dock(event):
    """Show or hide the Tabs dock."""
    c = event.get('c')
    dw = c and c.frame.top
    if not dw:
        return
    if not g.app.dock:
        g.es('this command requires docks')
        return
    dock = dw.tabs_dock
    if not dock:
        return
    if g.app.get_central_widget(c) == 'tabs':
        g.es('can not hide the central dock widget')
        return
    if dock.isVisible():
        dock.hide()
    else:
        dock.show()
#@+node:ekr.20190819091950.1: *4* qt_gui.create_dock_widget
total_docks = 0

def create_dock_widget(self, closeable, moveable, height, name):
    """Make a new dock widget in the main window"""
    dock = QtWidgets.QDockWidget(parent=self.main_window)
        # The parent must be a QMainWindow.
    features = dock.DockWidgetFloatable  # #1643.
    # #1643: Widgets are fixed unless --init-docks is in effect
    if moveable and g.app.init_docks:
        features |= dock.DockWidgetMovable
    if closeable:
        features |= dock.DockWidgetClosable
    dock.setFeatures(features)
    dock.setMinimumHeight(height)
    dock.setObjectName(f"dock-{self.total_docks}")
    self.total_docks += 1
    if name:
        dock.setWindowTitle(name.capitalize())
    else:
        # #1527. Suppress the title.
        w = QtWidgets.QWidget()
        dock.setTitleBarWidget(w)
    # #1327: frameFactory.createFrame now ensures that the main window is visible.
    return dock
#@+node:ekr.20200305075130.1: *4* qt_gui.find_dock
def find_dock(self, w):
    """return the QDockWidget containing w, or None"""
    dock = w
    while dock and not isinstance(dock, QtWidgets.QDockWidget):
        dock = dock.parent()
    return dock
#@+node:ekr.20190822113212.1: *4* qt_gui.make_global_outlines_dock
def make_global_outlines_dock(self):
    """
    Create the top-level Outlines (plural) dock,
    containing the 
    The dock's widget will be set later.
    """
    main_window = self.main_window
    # For now, make it the central widget.
    is_central = True
    dock = self.create_dock_widget(
        closeable=not is_central,
        moveable=not is_central,
        height=50,  # was 100: #1339.
        name='',  # #1527: was 'Leo Outlines'
    )
    if is_central:
        main_window.setCentralWidget(dock)
    else:
        area = QtCore.Qt.BottomDockWidgetArea
        main_window.addDockWidget(area, dock)
    return dock
#@+node:ekr.20190601054955.1: *4* qt_gui.raise_dock
def raise_dock(self, widget):
    """Raise the nearest parent QDockWidget, if any."""
    while widget:
        if isinstance(widget, QtWidgets.QDockWidget):
            widget.raise_()
            return
        if not hasattr(widget, 'parent'):
            return
        widget = widget.parent()
#@-all
#@@nosearch
#@-leo
