"""Detect all Python scripts in HTML pages in current folder and subfolders.
Generate brython_modules.js, a bundle with all the modules and packages used
by an application.
Generate a Python package ready for installation and upload on PyPI.
"""


import os
import shutil
import html.parser
import json
import traceback
import sys
import time
import io
import tokenize
import token

# Template for application setup.py script
setup = """from setuptools import setup, find_packages

import os

if os.path.exists('README.rst'):
    with open('README.rst', encoding='utf-8') as fobj:
        LONG_DESCRIPTION = fobj.read()

setup(
    name='{app_name}',
    version='{version}',

    # The project's main homepage.
    url='{url}',

    # Author details
    author='{author}',
    author_email='{author_email}',

    # License
    license='{license}',

    packages=['data'],
    py_modules=["{app_name}"],
    package_data={{'data':[{files}]}}
)
"""

# Template for the application script
app = """import os
import shutil
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--install',
    help='Install {app_name} in an empty directory',
    action="store_true")
args = parser.parse_args()

files = ({files})

if args.install:
    print('Installing {app_name} in an empty directory')

    src_path = os.path.join(os.path.dirname(__file__), 'data')

    if os.listdir(os.getcwd()):
        print('{app_name} can only be installed in an empty folder')
        import sys
        sys.exit()

    for path in files:
        dst = os.path.join(os.getcwd(), path)
        head, tail = os.path.split(dst)
        if not os.path.exists(head):
            os.mkdir(head)
        shutil.copyfile(os.path.join(src_path, path), dst)

"""


class FromImport:

    def __init__(self):
        self.source = ''
        self.type = "from"
        self.level = 0
        self.expect = "source"
        self.names = []

    def __str__(self):
        return '<import ' + str(self.names) + ' from ' + str(self.source) +'>'


class Import:

    def __init__(self):
        self.type = "import"
        self.expect = "module"
        self.modules = []

    def __str__(self):
        return '<import ' + str(self.modules) + '>'


class ImportsFinder:

    def __init__(self, *args, **kw):
        self.package = kw.pop("package") or ""
        
    def find(self, src):
        """Find imports in source code src. Uses the tokenize module instead
        of ast in previous Brython version, so that this script can be run
        with CPython versions older than the one implemented in Brython."""
        imports = set()
        importing = None
        f = io.BytesIO(src.encode("utf-8"))
        for tok_type, tok_string, *_ in tokenize.tokenize(f.readline):
            tok_type = token.tok_name[tok_type]
            if importing is None:
                if tok_type == "NAME" and tok_string in ["import", "from"]:
                    context = Import() if tok_string == "import" \
                        else FromImport()
                    importing = True
            else:
                if tok_type == "NEWLINE":
                    imports.add(context)
                    importing = None
                else:
                    self.transition(context, tok_type, tok_string)

        if importing:
            imports.add(context)

        # Transform raw import objects into a list of qualified module names
        self.imports = set()
        for imp in imports:
            if isinstance(imp, Import):
                for mod in imp.modules:
                    parts = mod.split('.')
                    while parts:
                        self.imports.add('.'.join(parts))
                        parts.pop()
            elif isinstance(imp, FromImport):
                source = imp.source
                if imp.level > 0:
                    if imp.level == 1:
                        imp.source = self.package
                    else:
                        parts = self.package.split(".")
                        imp.source = '.'.join(parts[:1 - imp.level])
                    if source:
                        imp.source += '.' + source
                parts = imp.source.split('.')
                while parts:
                    self.imports.add('.'.join(parts))
                    parts.pop()
                self.imports.add(imp.source)
                for name in imp.names:
                    parts = name.split('.')
                    while parts:
                        self.imports.add(imp.source + '.' + '.'.join(parts))
                        parts.pop()

    def transition(self, context, token, value):
        if context.type == "from":
            if token == "NAME":
                if context.expect == "source":
                    if value == "import" and context.level:
                        # syntax "from . import name"
                        context.expect = "names"
                    else:
                        context.source += value
                        context.expect = "."
                elif context.expect == "." and value == "import":
                    context.expect = "names"
                elif context.expect == "names":
                    context.names.append(value)
                    context.expect = ","
            elif token == "OP":
                if value == "," and context.expect == ",":
                    context.expect = "names"
                elif value == "." and context.expect == ".":
                    context.source += '.'
                    context.expect = "source"
                elif value == "." and context.expect == "source":
                    context.level += 1

        elif context.type == "import":
            if token == "NAME":
                if context.expect == "module":
                    if context.modules and context.modules[-1].endswith("."):
                        context.modules[-1] += value
                    else:
                        context.modules.append(value)
                    context.expect = '.'
            elif token == "OP":
                if context.expect == ".":
                    if value == ".":
                        context.modules[-1] += '.'
                    context.expect = "module"


class ModulesFinder:

    def __init__(self, directory=os.getcwd()):
        self.directory = directory
        self.modules = set()

    def get_imports(self, src, package=None):
        """Get all imports in source code src."""
        finder = ImportsFinder(package=package)
        finder.find(src)
        for module in finder.imports:
            if module in self.modules:
                continue
            found = False
            for module_dict in [stdlib, user_modules]:
                if module in module_dict:
                    found = True
                    self.modules.add(module)
                    if module_dict[module][0] == '.py':
                        is_package = len(module_dict[module]) == 4
                        if is_package:
                            package = module
                        elif "." in module:
                            package = module[:module.rfind(".")]
                        else:
                            package = ""
                        module_dict[module][2] = list(self.get_imports(
                            module_dict[module][1], package))
        return finder.imports

    def norm_indent(self, script):
        """Scripts in Brython page may start with an indent, remove it before
        building the AST.
        """
        indent = None
        lines = []
        for line in script.split('\n'):
            if line.strip() and indent is None:
                indent = len(line) - len(line.lstrip())
                line = line[indent:]
            elif indent is not None:
                line = line[indent:]
            lines.append(line)
        return '\n'.join(lines)

    def inspect(self):
        """Walk the directory to find all pages with Brython scripts, parse
        them to get the list of modules needed to make them run.
        """
        site_packages = 'Lib{0}site-packages{0}'.format(os.sep)
        imports = set()
        for dirname, dirnames, filenames in os.walk(self.directory):
            for name in dirnames:
                if name.endswith('__dist__') or name.endswith("__pycache__"):
                    # don't inspect files in the subfolder __dist__
                    dirnames.remove(name)
                    break
            for filename in filenames:
                path = os.path.join(dirname, filename)
                if path == __file__:
                    continue
                ext = os.path.splitext(filename)[1]
                if ext.lower() == '.html':
                    print("script in html", filename)
                    # detect charset
                    charset_detector = CharsetDetector()
                    with open(path, encoding="iso-8859-1") as fobj:
                        charset_detector.feed(fobj.read())

                    # get text/python scripts
                    parser = BrythonScriptsExtractor(dirname)
                    with open(path, encoding=charset_detector.encoding) as fobj:
                        parser.feed(fobj.read())
                    for script in parser.scripts:
                        script = self.norm_indent(script)
                        try:
                            self.get_imports(script)
                        except SyntaxError:
                            print('syntax error', path)
                            traceback.print_exc(file=sys.stderr)
                elif ext.lower() == '.py':
                    #print("python", filename)
                    if filename == "list_modules.py":
                        continue
                    if dirname != self.directory and not is_package(dirname):
                        continue
                    # get package name
                    package = dirname[len(self.directory) + 1:] or None
                    if package is not None and \
                            package.startswith(site_packages):
                        package = package[len('Lib/site-packages/'):]
                    with open(path, encoding="utf-8") as fobj:
                        try:
                            imports |= self.get_imports(fobj.read(), package)
                        except SyntaxError:
                            print('syntax error', path)
                            traceback.print_exc(file=sys.stderr)

    def make_brython_modules(self):
        """Build brython_modules.js from the list of modules needed by the
        application.
        """
        vfs = {"$timestamp": int(1000 * time.time())}
        for module in self.modules:
            dico = stdlib if module in stdlib else user_modules
            vfs[module] = dico[module]
            elts = module.split('.')
            for i in range(1, len(elts)):
                pkg = '.'.join(elts[:i])
                if not pkg in vfs:
                    vfs[pkg] = dico[pkg]
        # save in brython_modules.js
        path = os.path.join(stdlib_dir, "brython_modules.js")
        with open(path, "w", encoding="utf-8") as out:
            # Add VFS_timestamp ; used to test if the indexedDB must be
            # refreshed
            out.write("__BRYTHON__.VFS_timestamp = {}\n".format(
                int(1000 * time.time())))
            out.write("__BRYTHON__.use_VFS = true\nvar scripts = ")
            json.dump(vfs, out)
            out.write("\n__BRYTHON__.update_VFS(scripts)")


    def _dest(self, base_dir, dirname, filename):
        """Build the destination path for a file."""
        elts = dirname[len(os.getcwd()) + 1:].split(os.sep)
        dest_dir = base_dir
        for elt in elts:
            dest_dir = os.path.join(dest_dir, elt)
            if not os.path.exists(dest_dir):
                os.mkdir(dest_dir)
        return os.path.join(dest_dir, filename)

    def make_setup(self):
        """Make the setup script (setup.py) and the entry point script
        for the application."""
        # Create a temporary directory
        temp_dir = '__dist__'
        if os.path.exists(temp_dir):
            shutil.rmtree(temp_dir)
        os.mkdir(temp_dir)

        # Create a package "data" in this directory
        data_dir = os.path.join(temp_dir, 'data')
        os.mkdir(data_dir)
        with open(os.path.join(data_dir, "__init__.py"), "w") as out:
            out.write('')

        # If there is a brython_setup.json file, use it to get information
        if os.path.exists("brython_setup.json"):
            with open("brython_setup.json", encoding="utf-8") as fobj:
                info = json.load(fobj)
        else:
            # Otherwise, ask setup information
            while True:
                app_name = input("Application name: ")
                if app_name:
                    break
            while True:
                version = input("Version: ")
                if version:
                    break
            author = input("Author: ")
            author_email = input("Author email: ")
            license = input("License: ")
            url = input("Project url: ")
            info = {
                "app_name": app_name,
                "version": version,
                "author": author,
                "author_email": author_email,
                "license": license,
                "url": url
            }
            # Store information in brython_setup.json
            with open("brython_setup.json", "w", encoding="utf-8") as out:
                json.dump(info, out, indent=4)

        # Store all application files in the temporary directory. In HTML
        # pages, replace "brython_stdlib.js" by "brython_modules.js"
        files = []
        for dirname, dirnames, filenames in os.walk(self.directory):
            if dirname == "__dist__":
                continue
            if "__dist__" in dirnames:
                dirnames.remove("__dist__")
            for filename in filenames:
                path = os.path.join(dirname, filename)
                parts = path[len(os.getcwd()) + 1:].split(os.sep)
                files.append("os.path.join(" +
                             ", ".join(repr(part) for part in parts) +")")
                if os.path.splitext(filename)[1] == '.html':
                    # detect charset
                    charset_detector = CharsetDetector()
                    with open(path, encoding="iso-8859-1") as fobj:
                        charset_detector.feed(fobj.read())
                    encoding = charset_detector.encoding

                    # get text/python scripts
                    parser = VFSReplacementParser(dirname)
                    with open(path, encoding=encoding) as fobj:
                        parser.feed(fobj.read())
                    if not parser.has_vfs:
                        # save file
                        dest = self._dest(data_dir, dirname, filename)
                        shutil.copyfile(path, dest)
                        continue
                    with open(path, encoding=encoding) as fobj:
                        lines = fobj.readlines()
                        start_line, start_pos = parser.start
                        end_line, end_pos = parser.end
                        res = ''.join(lines[:start_line - 1])
                        for num in range(start_line - 1, end_line):
                            res += lines[num].replace("brython_stdlib.js",
                                "brython_modules.js")
                        res += ''.join(lines[end_line:])
                    dest = self._dest(data_dir, dirname, filename)
                    with open(dest, 'w', encoding=encoding) as out:
                        out.write(res)
                else:
                    dest = self._dest(data_dir, dirname, filename)
                    shutil.copyfile(path, dest)

        info["files"] = ',\n'.join(files)

        # Generate setup.py from the template in string setup
        path = os.path.join(temp_dir, "setup.py")
        with open(path, "w", encoding="utf-8") as out:
            out.write(setup.format(**info))

        # Generate the application script from the template in string app
        path = os.path.join(temp_dir, "{}.py".format(info["app_name"]))
        with open(path, "w", encoding="utf-8") as out:
            out.write(app.format(**info))

# User-defined Python modules and packages
user_modules = {}

# Get all modules in the Brython standard distribution.
# They must be in brython_stdlib.js somewhere in the current directory
# or below.
print('searching brython_stdlib.js...')
stdlib = {}
stdlib_dir = None
for dirname, dirnames, filenames in os.walk(os.getcwd()):
    for filename in filenames:
        if filename == "brython_stdlib.js":
            stdlib_dir = dirname
            path = os.path.join(dirname, filename)
            with open(path, encoding="utf-8") as fobj:
                modules = fobj.read()
                modules = modules[modules.find('{'):
                        modules.find('__BRYTHON__.update_VFS(')]
                stdlib = json.loads(modules)

if stdlib_dir is None:
    raise FileNotFoundError("Could not find brython_stdlib.js in this"
        " directory or below")
else:
    # search in site-packages
    sp_dir = os.path.join(stdlib_dir, "Lib", "site-packages")
    if os.path.exists(sp_dir):
        print("search in site-packages...")
        mf = ModulesFinder()
        for dirpath, dirnames, filenames in os.walk(sp_dir):
            if dirpath.endswith("__pycache__"):
                continue
            package = dirpath[len(sp_dir) + 1:]
            for filename in filenames:
                if not filename.endswith(".py"):
                    continue
                fullpath = os.path.join(dirpath, filename)
                #print(fullpath)
                is_package = False
                if not package:
                    # file in site-packages
                    module = os.path.splitext(filename)[0]
                else:
                    elts = package.split(os.sep)
                    is_package = filename == "__init__.py"
                    if not is_package:
                        elts.append(os.path.splitext(filename)[0])
                    module = ".".join(elts)
                with open(fullpath, encoding="utf-8") as f:
                    src = f.read()
                #imports = mf.get_imports(src)
                stdlib[module] = [".py", src, None]
                if is_package:
                    stdlib[module].append(1)

packages = {os.getcwd(), os.getcwd() + '/Lib/site-packages'}

def is_package(folder):
    """Test if folder is a package, ie has __init__.py and all the folders
    above until os.getcwd() also have __init__.py.
    Use set "packages" to cache results.
    """
    if folder in packages:
        return True
    current = folder
    while True:
        if not os.path.exists(os.path.join(current, "__init__.py")):
            return False
        current = os.path.dirname(current)
        if current in packages:
            packages.add(folder)
            return True

print('finding packages...')
for dirname, dirnames, filenames in os.walk(os.getcwd()):
    for filename in filenames:
        name, ext = os.path.splitext(filename)
        if not ext == ".py" or filename == "list_modules.py":
            continue
        if dirname == os.getcwd():
            # modules in the same directory
            path = os.path.join(dirname, filename)
            with open(path, encoding="utf-8") as fobj:
                src = fobj.read()
            mf = ModulesFinder(dirname)
            imports = sorted(list(mf.get_imports(src)))
            user_modules[name] = [ext, src, imports]
        elif is_package(dirname):
            # modules in packages below current directory
            path = os.path.join(dirname, filename)
            package = dirname[len(os.getcwd()) + 1:].replace(os.sep, '.')
            if package.startswith('Lib.site-packages.'):
                package = package[len('Lib.site-packages.'):]
            if filename == "__init__.py":
                module_name = package
            else:
                module_name = "{}.{}".format(package, name)
            with open(path, encoding="utf-8") as fobj:
                src = fobj.read()
            #mf = ModulesFinder(dirname)
            #imports = mf.get_imports(src, package or None)
            #imports = sorted(list(imports))
            user_modules[module_name] = [ext, src, None]
            if module_name == package:
                user_modules[module_name].append(1)


class CharsetDetector(html.parser.HTMLParser):
    """Used to detect <meta charset="..."> in HTML page."""

    def __init__(self, *args, **kw):
        kw.setdefault('convert_charrefs', True)
        try:
            html.parser.HTMLParser.__init__(self, *args, **kw)
        except TypeError:
            # convert_charrefs is only supported by Python 3.4+
            del kw['convert_charrefs']
            html.parser.HTMLParser.__init__(self, *args, **kw)

        self.encoding = "iso-8859-1"

    def handle_starttag(self, tag, attrs):
        if tag.lower() == "meta":
            for key, value in attrs:
                if key == "charset":
                    self.encoding = value

class BrythonScriptsExtractor(html.parser.HTMLParser):
    """Used to extract all Brython scripts in HTML pages."""

    def __init__(self, dirname, **kw):
        kw.setdefault('convert_charrefs', True)
        try:
            html.parser.HTMLParser.__init__(self, **kw)
        except TypeError:
            # convert_charrefs is only supported by Python 3.4+
            del kw['convert_charrefs']
            html.parser.HTMLParser.__init__(self, **kw)

        self.dirname = dirname
        self.scripts = []
        self.py_tags = [] # stack of Python blocks
        self.tag_stack = []

    def handle_starttag(self, tag, attrs):
        if tag.lower() == "script":
            _type = "js_script"
            src = None
            for key, value in attrs:
                if key == 'type' and value in ("text/python", "text/python3"):
                    _type = "py_script"
                elif key == "src":
                    src = value
            if _type == "py_script" and src:
                _type = "py_script_with_src"
                path = os.path.join(self.dirname, src)
                with open(path, encoding="utf-8") as fobj:
                    self.scripts.append(fobj.read())
            self.tag_stack.append(_type)

    def handle_endtag(self, tag):
        if tag.lower() == "script":
            self.tag_stack.pop()

    def handle_data(self, data):
        """Data is printed unchanged"""
        if data.strip():
            if self.tag_stack and self.tag_stack[-1].lower() == "py_script":
                self.scripts.append(data)


class VFSReplacementParser(html.parser.HTMLParser):
    """Used to replace brython_stdlib.js by brython_modules.js in HTML
    pages."""

    def __init__(self, path, **kw):
        kw.setdefault('convert_charrefs', True)
        try:
            html.parser.HTMLParser.__init__(self, **kw)
        except TypeError:
            # convert_charrefs is only supported by Python 3.4+
            del kw['convert_charrefs']
            html.parser.HTMLParser.__init__(self, **kw)
        self.vfs = False
        self.has_vfs = False

    def handle_starttag(self, tag, attrs):
        if tag.lower() == "script":
            _type = "js_script"
            src = None
            for key, value in attrs:
                if key == "src":
                    elts = value.split("/")
                    if elts and elts[-1] == "brython_stdlib.js":
                        self.vfs = True
                        self.has_vfs = True
                        self.attrs = attrs
                        self.start = self.getpos()
                        return
        self.vfs = False

    def handle_endtag(self, tag):
        if tag.lower() == "script" and self.vfs:
            self.end = self.getpos()


if __name__ == "__main__":
    finder = ModulesFinder()
    finder.inspect()
    print(sorted(list(finder.modules)))

