#!python


"""
Produce is a replacement for Make geared towards processing data rather than
compiling code. Key feature: supports multiple expansions in pattern rules.
https://github.com/texttheater/produce/
"""


import argparse
import ast
import collections
import concurrent.futures
import contextlib
import errno
import logging
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import tempfile
import threading
import time


### PLANNED FEATURES ##########################################################


# TODO includes
# TODO variation on the file rule type that unconditionally runs the recipe
# but with a temp file with target, then percolates dirtiness only if the new
# file differs from the existing version - e.g. for files that depend on data
# from a database.
# TODO rename private members


### UTILITIES #################################################################


def have_smart_terminal():
    # courtesy of apenwarr's redo
    return (os.environ.get('TERM') or 'dumb') != 'dumb'


def mtime(path, default=0):
    """
    Returns the mtime of the file at the specified path. Returns default if
    file does not exist. An OSError is raised if the file cannot be stat'd.
    """
    try:
        return os.stat(path).st_mtime
    except OSError as e:
        if e.errno == errno.ENOENT:
            return default
        raise e


def now():
    return time.time()


def remove_if_exists(path):
    try:
        os.remove(path)
    except OSError as e:
        if e.errno == errno.EISDIR:
            shutil.rmtree(path)
        elif e.errno == errno.ENOENT:
            pass  # Mission. F***ing. Accomplished.
        else:
            raise e


def rename_if_exists(src, dst):
    try:
        os.rename(src, dst)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e


def touch(path, timestamp):
    if subprocess.call(['touch', '-d', '@{}'.format(timestamp), path]) != 0:
        raise ProduceError('failed to touch {}'.format(path))


def shlex_join(value):
    return ' '.join((shlex.quote(str(x)) for x in value))


### GENERAL LOGGING ###########################################################


# General logging uses the root logger and configures it for each production
# (using the set_up_logging function). We use log levels INFO and DEBUG. For
# DEBUG messages, we also roll our own, finer-grained filter depending on how
# detailed debug output the user requested.


def set_up_logging(dbglvl):
    global debug_level
    debug_level = dbglvl
    if debug_level > 1:
        level = logging.DEBUG
    else:
        level = logging.INFO
    logging.basicConfig(format='%(levelname)s: %(message)s', level=level)


def debug(level, message_format, *parameters):
    if debug_level >= level:
        logging.debug(message_format, *parameters)


### LOGGING FOR STATUS MESSAGES ###############################################


# Status messages also use the logging module, but a different logger, called
# produce. Some unit tests hook into this logger using assertLogs to test that
# the right status messages are printed at the right time. The methods
# status_info and status_error are used to generate status messages.


class StatusMessage():

    def __init__(self, action, target, depth):
        self.action = action
        self.target = target
        self.depth = depth

    def __str__(self):
        return '{} {}'.format(self.action, self.target)


class StatusFormatter(logging.Formatter):

    def __init__(self, colors):
        logging.Formatter.__init__(self)
        if colors:
            self.red    = '\x1b[31m'
            self.green  = '\x1b[32m'
            self.yellow = '\x1b[33m'
            self.bold   = '\x1b[1m'
            self.plain  = '\x1b[m'
        else:
            self.red = ''
            self.green = ''
            self.yellow = ''
            self.bold = ''
            self.plain = ''

    def format(self, record):
        if isinstance(record.msg, StatusMessage):
            if record.levelno == logging.ERROR:
                color = self.red
            else:
                color = self.green
            return '{}{}{}{}{}{}{}'.format(
                color, record.msg.action,
                ' ' * (16 - len(record.msg.action)), self.bold,
                ' ' * record.msg.depth, record.msg.target, self.plain)
        else:
            return logging.Formatter.format(self, record)


# Set up logging for status messages:
status_logger = logging.getLogger('produce')
status_logger.propagate = False
status_logger.handlers = []  # remove default handler
status_logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
formatter = StatusFormatter(sys.stderr.isatty() and have_smart_terminal())
handler.setFormatter(formatter)
status_logger.addHandler(handler)


def status_info(message, target, depth):
    global status_logger
    status_logger.info(StatusMessage(message, target, depth))


def status_error(message, target, depth):
    global status_logger
    status_logger.error(StatusMessage(message, target, depth))


### ERRORS ####################################################################


class ProduceError(Exception):

    def __init__(self, message, cause=None):
        Exception.__init__(self, message)
        self.cause = cause


### COMMANDLINE PROCESSING ####################################################


def process_commandline(args=None):
    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        '-B', '--always-build', action='store_true',
        help="""Unconditionally build all specified targets and their
        dependencies""")
    group.add_argument(
        '-b', '--always-build-specified', action='store_true',
        help="""Unconditionally build all specified targets, but treat their
        dependencies normally (only build if out of date)""")
    parser.add_argument(
        '-d', '--debug', action='count', default=0,
        help="""Print debugging information. Give this option multiple times
        for more information.""")
    parser.add_argument(
        '-f', '--file', default='produce.ini',
        help="""Use FILE as a Producefile""")
    parser.add_argument(
        '-j', '--jobs', type=int, default=1,
        help="""Specifies the number of jobs (recipes) to run
        simultaneously""")
    parser.add_argument(
        '-n', '--dry-run', action='store_true',
        help="""Print status messages, but do not run recipes""")
    parser.add_argument(
        '-u', '--pretend-up-to-date', metavar='FILE', action='append',
        default=[],
        help="""Do not rebuild FILE or its dependencies (unless they are also
        depended on by other targets) even if out of date, but make sure that
        future invocations of Produce will still treat them as out of date by
        increasing the modification times of their changed dependencies as
        necessary.""")
    parser.add_argument(
        'target', nargs='*',
        help="""The target(s) to produce - if omitted, default target from
        Producefile is used""")
    return parser.parse_args(args)


### PRODUCEFILE PARSING #######################################################


SECTIONHEAD_PATTERN = re.compile(r'^\[(.*)\]\s*$')
AVPAIR_PATTERN = re.compile(r'^(\S+?)\s*=\s*(.*)$', re.DOTALL)
VALUECONT_PATTERN = re.compile(r'^(\s+)(.*)$', re.DOTALL)
COMMENT_PATTERN = re.compile(r'^\s*#.*$')
BLANKLINE_PATTERN = re.compile(r'^\s*$')


def parse_inifile(f):
    in_block = False
    current_name = None
    current_avpairs = None

    def current_section():
        return current_name, [(a, v.strip()) for (a, v) in current_avpairs]

    def extend_current_value(string):
        a, v = current_avpairs.pop()
        current_avpairs.append((a, v + string))

    for lineno, line in enumerate(f, start=1):
        match = SECTIONHEAD_PATTERN.match(line)
        if match:
            if in_block:
                yield current_section()
            in_block = True
            current_name = match.group(1)
            current_avpairs = []
            continue
        match = COMMENT_PATTERN.match(line)
        if match:
            continue
        if in_block:
            match = AVPAIR_PATTERN.match(line)
            if match:
                current_avpairs.append((match.group(1), match.group(2)))
                current_valuecont_pattern = VALUECONT_PATTERN
                continue
            if current_avpairs:
                match = current_valuecont_pattern.match(line)
                if match:
                    # Modify pattern for value continutation lines to cut off
                    # only as much whitespace as the first continutation line
                    # starts with:
                    current_valuecont_pattern = re.compile(
                        r'^(' + match.group(1) + r')(.*)$', re.DOTALL)
                    extend_current_value(match.group(2))
                    continue
        match = BLANKLINE_PATTERN.match(line)
        if match:
            if current_avpairs:
                extend_current_value(os.linesep)
            continue
        raise ProduceError('invalid line {} in Producefile'.format(lineno))
    if in_block:
        yield current_section()


def interpret_sections(sections):
    raw_globes = {}
    rules = []
    at_beginning = True
    for name, avpairs in sections:
        if at_beginning:
            at_beginning = False
            if name == '':
                raw_globes = collections.OrderedDict(avpairs)
                continue
        if name == '':
            raise ProduceError('The global section (headed []) is only allowed'
                               ' at the beginning of the Producefile.')
        else:
            rules.append((section_name_to_regex(name),
                          collections.OrderedDict(avpairs)))
    return raw_globes, rules


### PATTERN MATCHING AND INTERPOLATION ########################################


def interpolate(string, varz, ignore_undefined=False, keep_escaped=False):
    debug(4, 'interpolate called with varz: %s', {k: v for (k, v) in
          varz.items() if k != '__builtins__'})
    original_string = string
    result = ''
    while string:
        if string.startswith('%%'):
            if keep_escaped:
                result += '%%'
            else:
                result += '%'
            string = string[2:]
        elif string.startswith('%{'):
            # HACK: find the } that terminates the expansion by calling eval on
            # possible expressions of increasing length until we find one that
            # doesn't raise a syntax error. FIXME: things will still blow up if
            # the Python expression contains a comment that contains a }.
            start = 2
            error = None
            while '}' in string[start:]:
                debug(4, 'looking for } in %s, starting at %s', string, start)
                index = string.index('}', start)
                # Add parentheses so users don't need to add them for sequence
                # expressions:
                expression = '(' + string[2:index] + ')'
                try:
                    value = eval(expression, varz)
                    debug(4, 'interpolating %s into %s', expression, value)
                    result += value_to_string(value)
                    string = string[index + 1:]
                    break
                except SyntaxError as e:
                    error = e
                except NameError as e:
                    if ignore_undefined:
                        result += string[:index + 1]
                        string = string[index + 1:]
                        break
                    else:
                        raise ProduceError(
                            'name error in expression "{}" in string "{}": {}'
                            .format(expression, original_string, e))
                start = index + 1
            else:
                if error:
                    raise ProduceError(
                        '{} evaluating expression "{}" in pattern "{}": {}'
                        .format(error.__class__.__name__, expression,
                                original_string, error))
                else:
                    raise ProduceError(
                        'could not parse expression in string {}'
                        .format(original_string))
        elif string.startswith('%'):
            raise ProduceError(
                '% must be followed by % or variable name in curly braces in '
                'pattern {}'.format(original_string))
        else:
            result += string[:1]
            string = string[1:]
    return result


def value_to_string(value):
    if isinstance(value, str):
        return value
    try:
        return shlex_join(value)
    except TypeError:
        return str(value)


def section_name_to_regex(name, globes={}):
    if len(name) > 1 and name.startswith('/') and name.endswith('/'):
        try:
            return re.compile(name[1:-1])
        except Exception as e:
            raise ProduceError('{} in regex {}'.format(e, name))
    else:
        return produce_pattern_to_regex(name, globes)


def produce_pattern_to_regex(pattern, globes={}):
    ip = interpolate(pattern, globes, ignore_undefined=True, keep_escaped=True)
    regex = ''
    groups = set() # keep track of named groups
    while ip:
        if ip.startswith('%%'):
            regex += '%'
            ip = ip[:2]
        elif ip.startswith('%{') and '}' in ip:
            # TODO check that the part between curly braces is a valid
            # Python identifier (or, eventually, expression)
            index = ip.index('}')
            variable = ip[2:index]
            if variable in groups:
                # backreference to existing group
                regex += '(?P=' + variable + ')'
            else:
                # new named group
                regex += '(?P<' + variable + '>.*)'
                groups.add(variable)
            ip = ip[index + 1:]
        else:
            regex += re.escape(ip[:1])
            ip = ip[1:]
    regex += '$'  # re.match doesn't enforce reaching the end by itself
    debug(4, 'generated regex: %s', regex)
    return re.compile(regex)


### INSTANTIATED RULES ########################################################


# An instantiated rule is represented by a dict that corresponds to one rule
# section from the Producefile, with the values already expanded. There are two
# special keys: target (the target as matched by the section header), and type,
# which must be one of file and task and defaults to file.


def create_irule(target, rules, globes):
    debug(3, 'looking for rule to produce %s', target)
    # Go through rules until a pattern matches the target:
    for pattern, avdict in rules:
        match = pattern.match(target)
        if match:
            # Dictionary representing the instantiated rule:
            result = {}
            # Dictionary for local variables:
            # TODO is the empty string a good default?
            varz = dict(globes, **match.groupdict(default=''))
            # Special attribute: target
            result['target'] = target
            varz['target'] = target
            # Process attributes and their values:
            for attribute, value in avdict.items():
                # Remove prefix from attribute to get local variable name:
                loke = attribute.split('.')[-1]
                if loke == 'target':
                    raise ProduceError('cannot overwrite "target" attribute '
                                       'in rule for {}'.format(target))
                # Do expansions in value:
                iv = interpolate(value, varz)
                # Attribute retains prefix:
                result[attribute] = iv
                # Local variable does not retain prefix:
                varz[loke] = iv
                # If there is a condition and it isn't met, we stop processing
                # attributes so they don't raise errors:
                if attribute == 'cond' and not ast.literal_eval(iv):
                    break
            # If there is a condition and it isn't met, go to the next rule:
            if 'cond' in result and not ast.literal_eval(result['cond']):
                debug(3, 'condition %s failed, trying next rule',
                      result['cond'])
                continue
            debug(4, 'instantiated rule for target %s has varz: %s', target,
                  {k: v for (k, v) in varz.items() if k != '__builtins__'})
            result['type'] = result.get('type', 'file')
            result['jobs'] = result.get('jobs', '1')
            debug(3, 'target type: %s', result['type'])
            if result['type'] not in ('file', 'task'):
                raise ProduceError('unknown type {} for target {}'.format(
                    target_type, target))
            return result
        else:
            debug(3, 'pattern %s did not match, trying next rule', pattern)
    if os.path.exists(target):
        # Although there is no rule to make the target, the target is a file
        # that exists, so we can use it as an ingredient.
        return {'target': target, 'type': 'file', 'jobs': '1'}
    raise ProduceError('no rule to produce {}'.format(target))


def list_ddeps(irule):
    result = []
    for key, value in irule.items():
        if key.startswith('dep.'):
            result.append(value)
        elif key == 'deps':
            result.extend(shlex.split(value))
        elif key == 'depfile':
            try:
                result.extend(read_depfile(value))
            except IOError as e:
                raise ProduceError(
                    'cannot read depfile {} for target {}'
                    .format(value, irule['target']), e)
    return result


def list_outputs(irule):
    result = []
    for key, value in irule.items():
        if key.startswith('out.'):
            result.append(value)
        elif key == 'outputs':
            result.extend(shlex.split(value))
    return result


def read_depfile(filename):
    with open(filename) as f:
        return list(map(str.strip, f))


### PRODUCTION ################################################################


class ProductionResult:
    
    """The result of producing one target.

    Contains two fields: updated is True if the target was updated, False
    otherwise. mtime contains the target's modification time.
    """

    def __init__(self, updated, mtime):
        self.updated = updated
        self.mtime = mtime

    def __repr__(self):
        return 'ProductionResult({}, {})'.format(self.updated, self.mtime)


class Production:

    def __init__(self, rules, globes, dry_run, always_build,
                 always_build_these, jobs, pretend_up_to_date):
        self.rules = rules
        self.globes = globes
        self.dry_run = dry_run
        self.always_build = always_build
        self.always_build_these = always_build_these
        self.jobs = jobs
        self.semaphore = threading.BoundedSemaphore(jobs)
        self.sema_lock = threading.Lock() # controls access to semaphore
        self.pretend_up_to_date = pretend_up_to_date
        self.lock = threading.Lock() # controls access to certain fields
        self.target_locks = collections.defaultdict(threading.Lock)

    def produce(self, targets):
        self.exception = None
        self.target_result = {} # maps done targets to a ProductionResult or an exception
        try:
            results = self.produce_targets(targets, [], [], False)
            if any(result.updated for result in results):
                pass
            else:
                logging.info('all targets are up to date')
        except BaseException:
            # The exception we catch is not necessarily the one that was first
            # raised - that we store in the exception field.
            raise self.exception

    def register_exception(self, exception):
        """Register an exception that was raised in one of the worker threads.

        This method is thread-safe. The first registered exception will be
        stored to be re-raised by the produce method in the end. Storing an
        exception will trigger shutdown, i.e., the worker threads notice it and
        abort as soon as they can. Any further registered exceptions are
        ignored.
        """
        with self.lock:
            if self.exception is None:
                self.exception = exception

    def create_irule(self, target):
        return create_irule(target, self.rules, self.globes)

    def get_target_lock(self, target):
        with self.lock:
            return self.target_locks[target]

    def get_result(self, target):
        with self.lock:
            return self.target_result.get(target, None)

    def set_result(self, target, result):
        with self.lock:
            self.target_result[target] = result

    def pretend_up_to_date_for(self, target):
        return target in self.pretend_up_to_date

    def always_build_for(self, target):
        return self.always_build or (target in self.always_build_these)

    def is_shutting_down(self):
        return self.exception is not None

    def is_dry_run(self):
        return self.dry_run

    def produce_targets(self, targets, beam, neutral_targets,
                        pretend_up_to_date):
        """Takes a list of targets and produces them.

        Returns a list of ProductionResult objects.
        
        The core logic is delegated to produce_target for each target, each in
        a separate worker thread. produce_targets is mainly responsible for
        managing the workers and for handling exceptions.

        produce_targets is recursively called from produce_target in worker
        threads for producing dependencies, so it must synchronize access to
        fields.
        """
        if not targets:
            return []
        exception = None
        with concurrent.futures.ThreadPoolExecutor(self.jobs) as executor:
            # TODO can we use a single global executor? Then we would be more
            # sure of never hitting the thread limit. Also we would not need
            # the build_sema anymore.
            futures = [executor.submit(self.produce_target, target, beam,
                                       neutral_targets, pretend_up_to_date)
                       for target in targets]
            for future in concurrent.futures.as_completed(futures):
                e = future.exception()
                if e is not None:
                    exception = e
                    self.register_exception(e)
        if exception is None:
            return [future.result() for future in futures]
        else:
            raise exception
                    
    def produce_target(self, target, beam, neutral_targets,
                       pretend_up_to_date):
        """Handles the core production logic for a single target.

        Returns a ProductionResult object.

        Threading logic is kept outside this method as far as possible. It's
        just an ordinary method that takes an argument and returns a result or
        raises an exception. It still has to synchronize access to fields
        because it runs inside worker threads.
        """

        # Step 0: abort if shutting down
        if self.is_shutting_down():
            raise ProduceError('shutting down')

        # Step 1: catch cycles
        if target in beam:
            raise ProduceError('cyclic dependency: {}'.format(' <- '.join(
                [target] + beam)))

        # Step 2: create instantiated rule
        irule = self.create_irule(target)

        # Step 3: determine side outputs
        outputs = list_outputs(irule)

        # Step 4: catch soft cycles
        # We don't want to allow a target to depend on a rule that will produce
        # it as a side output: the target would be built twice by the same
        # production, which is crazy and probably indicates a bug. There's an
        # exception though: if the target has an empty recipe (we call that a
        # "neutral target"), it may depend on a target that produces it as a
        # side output. This is useful because it allows side outputs to have
        # dependencies: they can then declare which target to produce to get
        # the side output.
        for output in outputs:
            if output in beam and output not in neutral_targets:
                raise ProduceError(
                    'cyclic dependency: {}; {} has {} as output'.format(
                        ' <- '.join([target] + beam), target, output))

        # Step 4: lock all outputs
        lockables = set(outputs)
        if 'recipe' in irule:
            lockables.add(target)
        else:
            lockables.discard(target) # in case it's in outputs
        lockables = sorted(lockables) # to prevent deadlocks
        with contextlib.ExitStack() as stack:
            # Acquire locks. Release is automatic on leaving the context of the
            # stack. We lock all outputs at this early stage (before we produce
            # dependencies) so we are sure never to even try to produce one
            # target from two different worker threads.
            for lockable in lockables:
                debug(3, 'locking %s', lockable)
                stack.enter_context(self.get_target_lock(lockable))
                debug(3, 'locked %s', lockable)

            # Step 5: abort if target already done
            result = self.get_result(target)
            debug(3, 'retrieved result for {}: {}'.format(target, result))
            if result is None:
                pass
            elif isinstance(result, ProductionResult):
                return result
            else:
                raise result

            # Step 6: do target and mark as done
            try:
                result = self.do_target(target, irule, outputs, beam,
                                        neutral_targets, pretend_up_to_date)
                self.set_result(target, result)
                debug(3, 'created result for {}: {}'.format(target, repr(result)))
                # Step 7: mark other outputs as done
                for output in outputs:
                    if output == target:
                        continue
                    output_result = ProductionResult(True, mtime(output))
                    self.set_result(output, output_result)
                    debug(3, 'created result for {}: {}'.format(output, repr(output_result)))
                return result
            except BaseException as e:
                self.set_result(target, e)
                raise e
            
    def do_target(self, target, irule, outputs, beam, neutral_targets,
            pretend_up_to_date):
        # Step 0: update beam, neutral_targets, pretend_up_to_date
        beam = [target] + beam
        if 'recipe' not in irule:
            neutral_targets = [target] + neutral_targets
        pretend_up_to_date = pretend_up_to_date or \
                self.pretend_up_to_date_for(target)

        # Step 1: make depfile up to date, if any
        if 'depfile' in irule:
            self.produce_targets([irule['depfile']], beam, neutral_targets,
                                 pretend_up_to_date)

        # Step 2: determine direct dependencies
        ddeps = list_ddeps(irule)

        debug(2, '%s <- %s', target, ', '.join(ddeps))

        # Step 3: make dependencies up to date
        results = self.produce_targets(ddeps, beam,
            neutral_targets, pretend_up_to_date)

        # Step 4: determine if target is out of date
        out_of_date = False
        if irule['type'] == 'task':
            debug(2, '%s is out of date because it is a task', target)
            out_of_date = True
        elif self.always_build_for(target):
            debug(2, '%s is out of date because it is set to always build', target)
            out_of_date = True
        elif irule['type'] == 'file' and not os.path.exists(target):
            debug(2, '%s is out of date because it is a file and does not exist', target)
            out_of_date = True
        else:
            for ddep, result in zip(ddeps, results):
                if result.updated:
                    debug(2, '%s is out of date because its direct dependency %s was updated', target, ddep)
                    out_of_date = True
                    break
                elif result.mtime > mtime(target):
                    debug(2, '%s is out of date because its direct dependency %s is newer', target, ddep)
                    out_of_date = True
                    break

        # Step 5: abort if up to date or pretending
        if (not out_of_date) or pretend_up_to_date:
            return ProductionResult(False, mtime(target))

        # Step 6: run recipe with semaphore bounding parallel recipes
        to_hold = min(self.jobs, int(irule['jobs']))
        holding = 0
        # Acquire semaphore to_hold times. We use sema_lock to make sure only
        # one thread tries to acquire semaphore its needed number of times at a
        # time. Not sure this is necessary, but there miiight be livelock
        # situations without this precaution.
        self.sema_lock.acquire()
        while holding < to_hold:
            if not self.semaphore.acquire(timeout=0.1):
                # Timeout - release everything, sleep, try again.
                # If we didn't do this and held on to the holds we got, there
                # could be deadlocks.
                while holding > 0:
                    self.semaphore.release()
                    holding -= 1
                self.sema_lock.release()
                time.sleep(0.1)
                self.sema_lock.acquire()
            else:
                holding += 1
        self.sema_lock.release()
        # run recipe, then release semaphore to_hold times
        try:
            self.run_recipe(target, irule, outputs, len(beam) - 1)
            return ProductionResult(True, mtime(target))
        finally:
            while holding > 0:
                self.semaphore.release()
                holding -= 1

    def run_recipe(self, target, irule, outputs, depth):
        # Step 1: abort if shutting down
        if self.is_shutting_down():
            raise ProduceError('aborting due to shutdown')

        # Step 2: abort if no recipe
        if not 'recipe' in irule:
            return

        # Step 3: initial status info
        if irule['type'] == 'task':
            status_info('running task', target, depth)
        else:
            if os.path.exists(target):
                status_info('rebuilding file', target, depth)
            else:
                status_info('building file', target, depth)

        # Step 4: preprocess recipe and determine executable
        recipe = irule['recipe']
        executable = irule.get('shell', 'bash')
        if recipe.startswith('\n'):
            recipe = recipe[1:]

        # Step 5: optionally print recipe
        if debug_level >= 1:
            print(recipe)

        # Step 6: abort if dry run
        if self.is_dry_run():
            return

        # Step 7: remove old backup files, if any
        if irule['type'] == 'file':
            backup_name = target + '~'
            remove_if_exists(backup_name)
        for output in outputs:
            backup_name = output + '~'
            remove_if_exists(backup_name)

        # Step 8: create recipe file
        with tempfile.NamedTemporaryFile(mode='w') as recipefile:
            recipefile.write(recipe)
            recipefile.flush()

            # Step 9: run the recipe; try-finally for cleanup
            success = False
            try:
                proc = subprocess.Popen([executable, recipefile.name])
                debug(3, 'started subprocess')
                while True:
                    try:
                        debug(5, 'waiting')
                        proc.wait(0.05)
                    except subprocess.TimeoutExpired:
                        pass
                    if proc.returncode is not None:
                        break
                    if self.is_shutting_down():
                        proc.kill() # FIXME doesn't always kill all child processes
                if proc.returncode == 0:
                    success = True
                else:
                    raise ProduceError('recipe failed: {}'.format(repr(recipe)))
            finally:
                if success:
                    status_info('complete', target, depth)
                else:
                    if irule['type'] == 'file':
                        backup_name = target + '~'
                        debug(2, 'renaming %s to %s', target, backup_name)
                        rename_if_exists(target, backup_name)
                    for output in outputs:
                        backup_name = output + '~'
                        debug(2, 'renaming %s to %s', output, backup_name)
                        rename_if_exists(output, backup_name)
                    status_error('incomplete', target, depth)


### API #######################################################################


# TODO the API mimics the CLI, should maybe provide a better interface using
# kwargs


def produce(args=[]):
    args = process_commandline(args)
    set_up_logging(args.debug)
    try:
        with open(args.file) as f:
            sections = list(parse_inifile(f))
            debug(3, 'parsed sections: %s', sections)
            raw_globes, rules = interpret_sections(sections)
    except IOError as e:
        raise ProduceError('cannot read file {}'.format(args.file), e)
    globes = {}
    for att, val in raw_globes.items():
        globes[att] = interpolate(val, globes)
    # Determine targets:
    targets = args.target
    if not targets:
        if 'default' in globes:
            targets = shlex.split(globes['default'])
        else:
            raise ProduceError(
                "Don't know what to produce. Specify a target on the command "
                "line or a default target in {}.".format(args.file))
    # Add command-line targets to set of targets to build unconditionally, if
    # desired:
    if args.always_build_specified:
        always_build_these = set(targets)
    else:
        always_build_these = set()
    # Execute prelude, with globes providing the name context:
    exec(globes.get('prelude', ''), globes)
    # Produce:
    production = Production(rules, globes, args.dry_run, args.always_build,
                            always_build_these, args.jobs,
                            args.pretend_up_to_date)
    if _handle_signals: # HACK, see comment below
        def handler(signum, frame):
            production.register_exception(ProduceError('killed'))
        signal.signal(signal.SIGINT, handler)
        signal.signal(signal.SIGHUP, handler)
        signal.signal(signal.SIGTERM, handler)
    production.produce(targets)


### CLI #######################################################################


# When using Produce via the CLI, it installs signal handlers to enable a
# clean shutdown when it is killed. This is not possible when called via API,
# because signal handlers can only be installed from the main thread, and
# callers might have their own signal handling strategies. So the signal
# handling code should go here, except that the handler needs the Production
# object. So we put the code into the produce method above; here we only set a
# flag telling it whether to install the handlers or not. It's a bit messy,
# should be cleaned up once we refactor the API.
_handle_signals = False


if __name__ == '__main__':
    try:
        _handle_signals = True
        produce(None)
    except ProduceError as e:
        logging.error(e)  # FIXME prints only first line of error message???
        sys.exit(1)
