#!python

import sys
import os
import getopt
import shutil
import time
import ConfigParser
from datetime import datetime
from subprocess import Popen, PIPE
from esgcet.exceptions import *
from esgcet.model import *
from sqlalchemy import create_engine, func
from sqlalchemy.orm import sessionmaker
from esgcet.messaging import debug, info, warning, error, critical, exception
from esgcet.config import loadConfig, getHandler, getHandlerByName, initLogging, registerHandlers, getOfflineLister, splitLine

usage = """Usage:
    esgfind_excludes [options] directory [directory ...]

    Scan a set of directories recursively for a specific project, producing a list of thredds_exclude_variables. Works only for files matching filename variable_*_*.nc

Arguments:
    directory: A directory to scan.

Options:

    -h, --help: Print a help message.

    -i init_dir:
        Directory containing all initialization files.
        Recommended: one initialization file for the default sections (esg.ini) and one per project, must match the name format esg.<project>.ini
        If not specified, the default installed init files are read.

    --project project_name: Project identifier. Mandatory parameter.

    -u, --update-init: Updates thredds_exclude_variables in esg.<project>.ini (Please note: All comments in esg.ini will be deleted!)
"""

def main(argv):
    try:
        args, lastargs = getopt.getopt(argv, "hi:u", ['update-init', 'help', 'project='])
    except getopt.error:
        print sys.exc_value
        print usage
        sys.exit(0)

    if len(lastargs)==0:
        print 'No directory specified'
        print usage
        sys.exit(0)

    update = False
    init_dir = '/esg/config/esgcet/'
    project_name = None

    for flag, arg in args:
        if flag in ['-h', '--help']:
            print usage
            sys.exit(0)
        elif flag=='-i':
            init_dir = arg
        elif flag=='--project':
            project_name = arg
        elif flag in ['-u', '--update-init']:
            update = True

    # find exclude variables
    if project_name is None:
        print 'No project_id specified'
        print usage
        sys.exit(0)

    # Load the configuration file and the DB Session
    config = loadConfig(init_dir)
    dburl = config.getdburl('extract')
    engine = create_engine(config.getdburl('extract'), echo=False, pool_recycle=3600)
    initLogging('DEFAULT', override_sa=engine)
    Session = sessionmaker(bind=engine, autoflush=True, autocommit=False)

    project_init_file = os.path.join(init_dir, 'esg.%s.ini'%project_name)
    projectConfig = ConfigParser.ConfigParser()
    projectConfig.read(project_init_file)

    if not projectConfig:
        raise ESGPublishError('Configuration file parsing failed for esg.%s.ini'%project_name)

    # Register project handlers and find handler
    registerHandlers()
    handler = getHandlerByName(project_name, None, Session)

    if update and not os.access(project_init_file, os.W_OK): 
        warning('No write permissions for %s, can not update esg.%s.ini. Will output the list of variables instead.'%(project_init_file, project))
        update = False
    # try to backup old esg.ini
    if update:
        try:
            init_file_bak = project_init_file + '_' + time.strftime("%Y%m%d") + '.bak'
            shutil.copy(project_init_file, init_file_bak)
        except:
            warning('Could not create backup of esg.ini.')

    excludes = set()
    all_vars = set()
    var_and_exclude = set()
    cur_exclude_list = set(config.get('project:' + project_name, 'thredds_exclude_variables', default=None).replace(' ' , '').split(','))
    move_vars = False

    for root, subdirs, files in os.walk(lastargs[0]):
        if files:
            info("Scanning directory %s"%root)
        for filename in files:
            variable = filename.split('_')[0]
            # make sure target variable is not excluded
            if variable in cur_exclude_list:
                cur_exclude_list.difference_update(set([variable]))
                var_and_exclude.add(variable)
                move_vars = True
                info('Target-Variable %s currently in thredds_exclude_variables, delete from excludes and move to variable_locate.'%variable)

            all_vars.add(variable)

            # find all variables in file which are not target variable    
            path = os.path.join(root, filename)
            if path.endswith('.nc'):
                try:
                    openfile = handler.openPath(path)
                    for varname in openfile.inquireVariableList():
                        if varname != variable:
                            excludes.add(varname)
                    openfile.close()
                except:
                    warning("Cannot open %s, skipping ..."%path)
            else:
                info("No *.nc file: %s, skipping ..."%path)

    # generate new exclude lists and update esg.ini
    new_vars = excludes.difference(all_vars)-cur_exclude_list
    new_exclude_list = sorted(cur_exclude_list.union(new_vars))
    var_and_exclude.update(set(new_exclude_list).intersection(all_vars))

    if not var_and_exclude and not new_vars and not move_vars:
        info('Your thredds_exclude_variables and variable_locate lists are up-to-date :)')

    else:
        if var_and_exclude:
            varlocate = config.get('project:' + project_name, 'variable_locate', default=None)
            for var in var_and_exclude:
                if varlocate:
                    if var not in varlocate:
                        varlocate += ' | %s,_%s'%(var, var)
                else:
                    varlocate = '%s,_%s'%(var, var)

            info('Found new set of variable_locate variables: %s'%varlocate)
            if update:
                config.set('project:' + project_name, 'variable_locate', varlocate)

        if new_vars or move_vars:
            info('Found new set of thredds exclude variables: %s'%', '.join(new_exclude_list))
            if update:
                config.set('project:' + project_name, 'thredds_exclude_variables', ', '.join(new_exclude_list))

        if update:
            with open(project_init_file, 'wb') as configfile:
                info('Updating esg.ini ...')
                config.write(configfile)


if __name__=='__main__':
    main(sys.argv[1:])