#!python

import sys
import os
import getopt
import textwrap
import re
import getpass
from ConfigParser import RawConfigParser, SafeConfigParser, DEFAULTSECT, NoOptionError
import subprocess
import urlparse
import socket
from esgcet.exceptions import *

usage = """Usage:
    esgsetup --config [--rootid root_identifier]
    esgsetup --db [--db-admin admin] [--db-host hostname] [--db-port port] [--db-admin-password password] [--db-user-password password] [--db-name name]
    esgsetup --thredds [--thredds-password password]
    esgsetup --publish [--index index_node_host]
    esgsetup --handler

    Setup the ESG publication configuration.

Options:

    -h, --help: Help.

    --config: Create an initial ESG configuration file (esg.ini).
        Also see --rootid.

    --db: Initialize the database.

    --db-admin admin: Name of database administrator. Defaults to $PGUSER.

    --db-admin-password password: Password for database administrator. If not specified, the
        value is requested interactively. Only used with the --db option.

    --db-host hostname: Hostname for database connection. Defaults to $PGHOST, or 'localhost' if not set.

    --db-name name: Name of the database to create or access. Defaults to 'esgcet'.

    --db-port port: Port used for database connection. Defaults to $PGPORT, or 5432 if not set.

    --db-user user: Name of the (non-administrative) owner of the esgcet database.

    --db-user-password password: Password for non-administrative database user, that owns the
        esgcet database. If not specified, the value is requested interactively. Only used
        with the --db option.

    --handler
        Create customized handlers. Turns off minimal setup.

    --index index_node_host:
        Index node host name. Used to set default publication service endpoints.
        If not set, the user is prompted for service endpoints.
        Note: if environment variable ESG_GATEWAY_SVC_ROOT is set to the gateway
        service root, and this option is not used, the host name will be derived
        from the environment variable.

    --publish: Initialize publication-related configuration options.

    --rootid root_identifier:
        Initialize the ESG root identifier during the --config phase.

    --minimal-setup: Ask the minimum number of questions. May be used with any other options.

    --thredds: Initialize the THREDDS server.

    --thredds-password password: Password for TDS reinitialization. If not specified, the
        value is requested interactively. Only used with the --thredds option.

"""

PUBLISH_SERVICE = "remote/secure/client-cert/hessian/publishingService"
REST_WEB_SERVICE = "ws"
SOLR_SEARCH_SERVICE = "search"
METADATA_SERVICE = "remote/hessian/guest/remoteMetadataService"
REPOS = "http://www-pcmdi.llnl.gov/svn/repository/externals"
SERVICE_ROOT = "pcmdi9.llnl.gov/esg-search" # Override by setting ESG_GATEWAY_SVC_ROOT env variable
DEFAULT_COMPOUND_FILE_SERVICE = 'fileservice'
urlpat = r'(.*)://(.*):(.*)@([^:]*)(:\d+)?/(.*)'
minimalSetup = False                   # If True, ask the minimum number of questions


def askyn(question, default='y'):
    if minimalSetup:
        return (default=='y')
    answer = raw_input("%s [%s]: "%(question, default))
    if answer=='':
        result = default
    else:
        result = answer[0].lower()
    return (result=='y')


def askfor(question, default=None, env=None, silent=False, ignore_minimal=False):
    """
    question: String prompt
    default: String to return if a blank string is returned
    env: Get the default from the environment.
    silent: Don't echo (for passwords)
    ignore_minimal: ask even if minimal setup is set
    """
    if default is None and env is not None:
        default = os.environ.get(env)
    if minimalSetup and (not ignore_minimal) and default is not None:
        return default
    if not silent:
        asker = raw_input
    else:
        asker = getpass.getpass
    if default is None:
        answer = asker("%s: "%(question,))
        return answer
    else:
        answer = asker("%s [%s]: "%(question, default))
        if answer=='':
            answer = default
        return answer


def askForMultiple(prompt, displayCurrent=None):
    print "%s:"%prompt
    if displayCurrent is not None:
        print "Current value is: \n%s\n"%displayCurrent
        result = displayCurrent+'\n'
    else:
        result = ''
    option = askfor("Replace (r) or Add lines (a)?", default='a')
    if option.lower()[0]=='r':
        result = ''
    print "Enter lines, or <RETURN> to end"
    #while True:
    #    answer = raw_input("Add new line: ")
    #    if answer!='':
    #        result += answer+'\n'
    #    else:
    #        break
    return result


def tell(info):
    info = re.sub(r'\n[\t\r\f\v ]+', '\n   ', info)
    print info


def getconfignames():
    """Create esg.ini as:
    (1) $ESGINI if it exists
    (2) /esg/config/esgcet/esg.ini otherwise
    """
    configname = os.environ.get('ESGINI')
    if configname is not None:
        if os.path.exists(configname) and not os.path.isfile(configname):
            raise RuntimeError("$ESGINI = %s, is not a file"%configname)
        configdir = os.path.dirname(configname)
    else:
        configdir = '/esg/config/esgcet'
        configname = os.path.join(configdir, 'esg.ini')
    return configdir, configname


def updateconfig(updates, currentConfig=None):
    configdir, configname = getconfignames()
    if currentConfig is None:
        tell("Updating configuration file %s"%configname)
        config = SaneConfigParser(configname)
        config.read(configname)
    else:
        config = currentConfig
    for update in updates:
        apply(config.set, update)
    esgini = open(configname, 'w+')
    config.write(esgini)
    esgini.close()


def setupconfig():
    tell("""

    Create initial configuration file and fetch project configuration files:

    esg.ini is a text file that contains default configuration for ESGF publication
    esg.<project>.ini files are configuration files to publish individual projects

    The location of those files is set to

    (a) the location of environment variable $ESGINI if set, or
    (b) /esg/config/esgcet

    if it does not already exist.

    """)

    # Create .esgcet directory.
    configdir, configname = getconfignames()
    if not os.path.exists(configdir):
        tell("Creating %s"%configdir)
        os.mkdir(configdir, 0755)

    # Does esg.ini exist? If so, ask to proceed.
    templatename = os.path.join(configdir, 'template.ini')
    if os.path.exists(configname):
        if not askyn("Configuration file %s exists, replace?"%configname, default='n'):
            return

    # Copy template.ini.
    tell("Copying template.ini to %s"%configname)
    from pkg_resources import resource_string, cleanup_resources
    templatestring = resource_string('esgcet.config.etc', 'template.ini')
    esgini = open(configname, 'w')
    esgini.write(templatestring)
    esgini.close()
    os.chmod(configname, 0600)
    templateini = open(templatename, 'w')
    templateini.write(templatestring)
    templateini.close()

    # Copy model initialization file
    modelInitName = os.path.join(configdir, 'esgcet_models_table.txt')
    if not os.path.exists(modelInitName):
        tell("Copying esgcet_models_table to %s"%configdir)
        modelinitstring = resource_string('esgcet.config.etc', 'esgcet_models_table.txt')
        modelini = open(modelInitName, 'w')
        modelini.write(modelinitstring)
        modelini.close()

    # Copy standard name table
    standardNameInitName = os.path.join(configdir, 'cf-standard-name-table.xml')
    if not os.path.exists(standardNameInitName):
        tell("Copying cf-standard-name-table.xml to %s"%configdir)
        standard_nameinitstring = resource_string('esgcet.config.etc', 'cf-standard-name-table.xml')
        standard_nameini = open(standardNameInitName, 'w')
        standard_nameini.write(standard_nameinitstring)
        standard_nameini.close()

    cleanup_resources()
    return configdir


def fetch_project_ini():
    # Fetch new esg.<project>.ini files, do not overwrite existing files
    tell("""
    Fetching new esg.<project>.ini files from github:
    Running 'esgfetchini' -k:
    """)
    try:
        subprocess.call(["esgfetchini", "-k"])
    except OSError, e:
        raise OSError("Cannot run esgfetchini: %s. Is $CDAT_HOME/bin in the path?" % `e`)


def setupHandler(packageDir, packageName, option, handlerList, handlerEntryPoints, optionList, handlerGroup, handlerClass, handler=None, handlerBasename=None, handlerTemplate=None, handlerOption=None, handlerPath=None):
    from pkg_resources import resource_string

    entryPointTemplate = """
    [%s]
    handler_name = %s
    handler = %s
    """
    if handler is None:
        handler = "%s.%s_handler:%s"%(packageName, option, handlerClass)
    if handlerBasename is None:
        handlerBasename = "%s_handler.py"%option
    if handlerTemplate is None:
        handlerTemplate = "%s_handler.py.tmpl"%option
    if handlerOption is None:
        handlerOption = "%s_handler_name"%option
    if handlerPath is None:
        handlerPath = os.path.join(packageDir, handlerBasename)
    createit = True
    if os.path.exists(handlerPath):
        createit = askyn('Replace existing file %s?'%handlerPath)
    if createit:
        handlerString = resource_string('esgcet.config.etc', handlerTemplate)
        f = open(handlerPath, 'w')
        f.write(handlerString)
        f.close()

    handlerList += '\n' + handlerPath
    handlerName = askfor("What value will be used for the esg.ini %s option?"%handlerOption)
    entryPointSection = entryPointTemplate%(handlerGroup, handlerName, handler)
    handlerEntryPoints += entryPointSection
    optionList += "\n%s = %s"%(handlerOption, handlerName)
    return handlerList, handlerEntryPoints, optionList


# Note: this is duplicated from esgcet.config, but we don't know that it can be
# imported yet.
class SaneConfigParser(SafeConfigParser):

    def __init__(self, name, defaults=None):
        SafeConfigParser.__init__(self, defaults=defaults)
        self.name = name

    def get(self, section, option, raw=False, vars=None, **options):
        if vars is None:
            vars = {'here' : os.getcwd()}
        elif not vars.has_key('here'):
            vars['here'] = os.getcwd()

        home = os.environ.get('HOME')
        if home is not None:
            vars['home'] = home

        pythonbin = os.path.join(sys.prefix, 'bin')
        vars['pythonbin'] = pythonbin

        try:
            value = SafeConfigParser.get(self, section, option, raw=raw, vars=vars)
        except NoOptionError:
            if options.has_key('default'):
                value = options['default']
            else:
                raise ESGPublishError("Configuration file option missing: %s in section: %s, file=%s"%(option, section, self.name))
        return value

    def getboolean(self, section, option, default=None):
        try:
            value = SafeConfigParser.getboolean(self, section, option)
        except AttributeError:
            value = default
        return value

    def getint(self, section, option, default=None):
        try:
            value = SafeConfigParser.getint(self, section, option)
        except AttributeError:
            value = default
        return value

    def write(self, fp):
        """Write an .ini-format representation of the configuration state.
        with sorting added.
        """
        if self._defaults:
            fp.write("[%s]\n" % DEFAULTSECT)
            defaultItems = self._defaults.items()
            defaultItems.sort()
            for (key, value) in defaultItems:
                fp.write("%s = %s\n" % (key, str(value).replace('\n', '\n\t')))
            fp.write("\n")
        for section in self._sections:
            fp.write("[%s]\n" % section)
            sectionItems = self._sections[section].items()
            sectionItems.sort()
            for (key, value) in sectionItems:
                if key != "__name__":
                    fp.write("%s = %s\n" %
                             (key, str(value).replace('\n', '\n\t')))
            fp.write("\n")

def main(argv):

    global minimalSetup

    try:
        args, lastargs = getopt.getopt(argv, "h", ['config', 'db', 'db-admin=','db-admin-password=', 'db-user=', 'db-user-password=', 'db-host=', 'db-port=', 'db-name=', 'gateway=', 'handler', 'help', 'index=', 'publish', 'rootid=', 'minimal-setup', 'thredds', 'thredds-password='])
    except getopt.error:
        print sys.exc_value
        print usage
        sys.exit(0)

    dbadmin = None
    dbdef = None
    dbDefault = "esgcet"
    dbpass = None
    dbuser = None
    esgusepass = None
    host = None
    port = None
    gatewayHost = None
    generateHandler = False
    initconfig = False
    initdb = False
    initthredds = False
    initpublish = False
    initroot = False
    threddsPassword = None
    for flag, arg in args:
        if flag in ['-h', '--help']:
            print usage
            sys.exit(0)
        elif flag=='--config':
            initconfig = True
        elif flag=='--db':
            initdb = True
        elif flag=='--db-admin':
            dbadmin = arg
        elif flag=='--db-admin-password':
            dbpass = arg
        elif flag=='--db-name':
            dbdef = arg
        elif flag=='--db-user':
            dbuser = arg
        elif flag=='--db-user-password':
            esgusepass = arg
        elif flag=='--db-host':
            host = arg
        elif flag=='--db-port':
            port = arg
        elif flag=='--gateway':
            gatewayHost = arg
        elif flag=='--handler':
            generateHandler = True
        elif flag=='--index':
            gatewayHost = arg
        elif flag=='--minimal-setup':
            minimalSetup = True
        elif flag=='--publish':
            initpublish = True
        elif flag=='--thredds':
            initthredds = True
        elif flag=='--thredds-password':
            threddsPassword = arg
        elif flag=='--rootid':
            initroot = True
            rootid = arg

    if not (initconfig or initdb or initthredds or initpublish or generateHandler):
        print 'No option specified.'
        print usage
        sys.exit(0)

    if generateHandler:
        minimalSetup = False

    # Config file initialization
    if initconfig:
        configdir = setupconfig()
        print("Note: project .ini files should have been fetched (using esgfetchini or manually via github.) prior to running esgsetup.  Not doing so results in undefined behavior.")
#        fetch_project_ini()

        if initroot:
            updateconfig([('DEFAULT', 'root_id', rootid)])

        if askyn("Should checksums be generated for all scanned files? (This will slow processing.)", default='n'):
            updateconfig([('DEFAULT', 'checksum', "md5sum | MD5")])
            tell("Ensure that md5sum is in your path.")

    # Database initialization
    if initdb:

        # Preconditions:
        # - database server is running
        # - user has database admin access
        # - config file is initialized

        tell("""

        Create and initialize ESG node database:

        The node database is a relational database that contains
        information scanned by ESG software. This section will:

        - Create a non-administrative ESG user.
        - Create the database.
        - Create and initialize the database tables.
        - Update the configuration file.

        Before proceeding, make sure that:

        - The database server is running, and you know the host and
          port on which it is listening;

        - You have an administrative database account with privilege
          to create new databases.

        """)

        if not askyn("Proceed?"):
            raise RuntimeError

        try:
            # Is the database running?
            tell("Testing the database connection:")
            try:
                import psycopg2
                from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
            except ImportError, e:
                tell("""

                The following error occurred:

                - %s

                The module psycopg2 is needed for connection to the
                database, and was not correctly installed or
                configured. It should have been installed
                automatically when esgcet was installed.

                If the error is 'Cannot open shared object':
                - Check that the environment variable LD_LIBRARY_PATH contains the postgres library directory.

                otherwise:
                - Rerun 'easy_install -f %s esgcet' and check that psycopg2 builds correctly.

                """%(`e`,REPOS))
                sys.exit(1)

            if dbadmin is None:
                dbadmin = askfor("Database administrator name", env='PGUSER')
            if dbpass is None:
                dbpass = askfor("Administrator password", silent=True)
            if host is None:
                host = askfor("Host name", default="localhost", env='PGHOST')
            if port is None:
                port = askfor("Database port", default="5432", env='PGPORT')

            connectString = "dbname='postgres' user='%s' password='%s' host='%s' port='%s'"%(dbadmin, dbpass, host, port)
            echoConnectString = "dbname='postgres' user='%s' password='...' host='%s' port='%s'"%(dbadmin, host, port)

            tell("Connecting to database with %s"%echoConnectString)
            try:
                conn = psycopg2.connect(connectString)
                cur = conn.cursor()
                tell("Connection succeeded")
            except:
                tell("Cannot connect to the database")
                sys.exit(1)

            # Update the configuration
            configdir, configname = getconfignames()
            if not os.path.exists(configname):
                configdir = setupconfig()

            config = SafeConfigParser()
            config.read(configname)

            # Create owner of esgcet database
            tell("""
            The next step is to create the esgcet database, and an ESG user:
            """)
            initdburl = config.get('DEFAULT', 'dburl')
            scheme = "postgresql"
            userdef = "esgcet"
            passdef = ""
            hostdef = host
            portdef = port
            dbdef2 = None               # Existing database 
            if initdburl is not None:
                m = re.match(urlpat, initdburl)
                if m is not None:
                    scheme, userdef, passdef, hostdef, portdef, dbdef2 = m.groups()
                    if portdef is not None:
                        portdef = portdef[1:]

            # Set database name to:
            # 1) --db-name option if set
            # 2) existing database value from dburl if present
            # 3) "esgcet" otherwise
            if dbdef is None:
                if dbdef2 is not None:
                    dbdef = dbdef2
                else:
                    dbdef = dbDefault

            if dbuser is not None:
                esguser = dbuser
            else:
                esguser = askfor("esgcet database user", default=userdef)

            # Does this user exist?
            cur.execute("select usename, usesuper, usecreatedb from pg_user where usename='%s'"%esguser)
            rows = cur.fetchall()
            if len(rows)==0:
                createuser = True
            else:
                dum, usesuper, usecreatedb = rows[0]
                if usesuper or usecreatedb:
                    tell("This is an administrative user, cannot be the esgcet user.")
                    conn.close()
                    raise RuntimeError
                createuser = False

            # Create the user if necessary
            if esgusepass is None:
                esgusepass = askfor("esgcet user password", silent=True)
                esgusepassagain = askfor("Confirm esgcet user password", silent=True)
                if esgusepass!=esgusepassagain:
                    tell("Passwords do not match")
                    raise RuntimeError

            if createuser:
                tell("Creating new database user %s"%esguser)
                cur.execute("create user %s with encrypted password '%s'"%(esguser, esgusepass))
                conn.commit()

            # Check the user can connect
            connectString2 = "dbname='postgres' user='%s' password='%s' host='%s' port='%s'"%(esguser, esgusepass, host, port)
            echoConnectString2 = "dbname='postgres' user='%s' password='...' host='%s' port='%s'"%(esguser, host, port)
            tell("Testing database connection with %s"%echoConnectString2)
            try:
                conn2 = psycopg2.connect(connectString2)
            except:
                tell("Cannot connect with this ESG username and password: %s"%echoConnectString2)
                raise RuntimeError
            tell("Connection succeeded.")
            conn2.close()

            # Create esgcet database
            dbname = askfor("Name of esgcet database to create", default=dbdef)
            cur.execute("select datname from pg_database")
            rows = cur.fetchall()
            if dbname in [item[0] for item in rows]:
                ans = askyn("Database %s already exists, should it be reused? (This will NOT delete existing data.)"%dbname)
                createdb = not ans
            else:
                createdb = True

            if createdb:
                tell("Creating database %s with owner=%s"%(dbname, esguser))
                conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
                try:
                    cur.execute("drop database if exists %s"%dbname)
                    cur.execute("create database %s with owner %s"%(dbname, esguser))
                except psycopg2.ProgrammingError, e:
                    print 'Cannot create database %s: %s'%(dbname, `e`)
                except psycopg2.OperationalError, e:
                    print 'Cannot create database %s: %s'%(dbname, `e`)
                    print 'Try restarting postgres, then rerun esgsetup --db'
                    sys.exit(0)

            conn.close()

            # Test sqlalchemy installation
            try:
                import sqlalchemy
            except ImportError, e:
                tell("""

                The following error occurred:

                - %s

                Module sqlalchemy has not been installed or configured
                correctly. SQLAlchemy is the object relational
                interface to the database. It should have been
                installed automatically when esgcet was installed.

                Rerun 'easy_install -f %s esgcet' and check that sqlalchemy built correctly.

                """%(`e`,REPOS))
                raise RuntimeError

            # SQLAlchemy versions <0.6 have protocol "postgres", >=0.6 use "postgresql"
            from sqlalchemy import __version__ as sqlalchemy_version
            if sqlalchemy_version >= '0.6':
                dbschema = 'postgresql'
            else:
                dbschema = 'postgres'

            dburl = dbschema+"://%s:%s@%s:%s/%s"%(esguser, esgusepass, host, port, dbname)
            updateconfig([('DEFAULT', 'dburl', dburl)])

            # Setup schema migration if necessary ...
            from pkg_resources import resource_filename
            repo_cfg_path = resource_filename('esgcet.schema_migration', 'migrate.cfg')
            repo_path = os.path.dirname(repo_cfg_path)

            try:
                from migrate.versioning.api import version_control, db_version, downgrade
            except ImportError, e:
                tell("""

                The following error occurred:

                - %s

                The sqlalchemy_migrate package has not been installed. It should have been built when
                the esgcet package was installed.

                Run 'easy_install SQLAlchemy-migrate' and ensure that the package built correctly.
                """%`e`)
                raise RuntimeError

            try:
                # >=0.6.1
                from migrate import DatabaseAlreadyControlledError
            except:
                # <=0.6.0
                from migrate.versioning.exceptions import DatabaseAlreadyControlledError

            # ... If the database was just created, then just initialize version control with version=0
            if createdb:
                try:
                    version_control(dburl, repo_path)
                    print 'Enabled schema version control.'
                except DatabaseAlreadyControlledError:
                    tell("""

                    This is unexpected: the database was just created, but appears to be under
                    version control already. Suggest running 'esgdrop_tables' then 'esginitialize -c'
                    to ensure that the database version is correct.

                    """)
                    raise RuntimeError

            # ... Otherwise the database already exists: check if version control is already enabled. If not,
            #     initialize with version=1 and downgrade to 0 to ensure that the
            #     database is at the correct version
            else:
                try:
                    version = db_version(dburl, repo_path)
                except:
                    # The database already exists, but schema versioning is not enabled.
                    tell("""

                    This database is not version controlled. Enabling version control ensures that:
                    - this version of esgcet is consistent with the database, and
                    - future esgcet package versions can be installed without rebuilding the database.

                    In order to initialize schema version control, all existing tables must be deleted
                    and rebuilt. THIS WILL DELETE ALL DATA IN THE NODE DATABASE. Once version control
                    is enabled, future upgrades will not require database deletion.

                    """)
                    ans = askyn("Do you want to rebuild the existing node database, and enable schema version control? (THIS WILL DELETE EXISTING DATA)")
                    if ans:
                        print 'Enabling schema version control ...'
                        version_control(dburl, repo_path, version=1)
                        downgrade(dburl, repo_path, 0)
                        print 'Schema version control enabled.'
                    else:
                        tell("""

                        Schema version control was NOT enabled. To ensure that the node database has the correct
                        schema version, please run 'esgdrop_tables' and 'esginitialize -c'.

                        """)
                        sys.exit(0)

            # Initialize database tables
            tell("""
            Checking that database %s is up-to-date:
            Running 'esginitialize -c':
            """%dbname)
            try:
                subprocess.call(["esginitialize", "-c"])
            except OSError, e:
                raise OSError("Cannot run esginitialize: %s. Is $CDAT_HOME/bin in the path?"%`e`)

        except RuntimeError:
            tell("Script exited.")
            sys.exit(1)

    # THREDDS initialization
    if initthredds:

        # Preconditions
        # - TDS is running
        # - user has r/w access to TDS content and conf directory
        # - TDS is setup for digest authentication, https
        # - user has entry and tdsConf role in tomcat-users.xml
        # - thredds.xml file exists for Tomcat V6

        tell("""

        Setup the interface to the THREDDS Data Server:

        TDS is the data server that is used for ESG publication. This
        section will:

        - Create a content directory where TDS catalogs will be
          written by the publisher.
        - Create a top-level ESG catalog, and add a link to the top TDS catalog.
        - Test TDS reinitialization. This makes the data visible
          through TDS without restarting the server.
        - Setup TDS service definitions. Examples of services are http, GridFTP, etc.  
        - Save TDS configuration options to esg.ini

        One of the TDS configuration options is thredds_dataset_roots. This is a list
        of top-level directories for which TDS will serve data. The rules for dataset root
        directories are:

        - A file not contained in any dataset root or subdirectory is not accessible (cannot be downloaded).
        - All published files under a dataset root or subdirectory are accessible and visible
          through THREDDS and the web portal.
        - All unpublished files contained in a dataset root or its subdirectories are potentially accessible,
          but not visible in THREDDS or the web portal.
        - The dataset root should not have any symbolic links to directories outside the dataset root.

        Before proceeding, ensure that:

        - Tomcat and TDS are installed and running

        - You have read/write access to the Tomcat content directory.

        - Tomcat and TDS are configured for https and digest
          authentication. For Tomcat V6, the file
          conf/Catalina/localhost/thredds.xml should exist.

        - You have a TDS username/password in tomcat-users.xml.  
        """)

        if not askyn("Proceed?"):
            raise RuntimeError

        try:

            maj, min, dum, dum, dum = sys.version_info
            if (maj, min) < (2,6):
                if not socket._have_ssl:
                    tell("""

                    The Python SSL module was not found. SSL is need to
                    communicate securely with TDS and the index node:
                    - Install OpenSSL on your system and rebuild Python.
                    - Rerun 'easy_install -f %s esgcet'.

                    """%REPOS)
                    sys.exit(1)

            try:
                import lxml.etree
            except ImportError:
                tell("""

                The module lxml is needed for connection to the database, and was not correctly installed.
                It should have been installed automatically when esgcet was installed.

                Rerun 'easy_install -f %s esgcet' and check that lxml builds correctly.

                """%REPOS)
                sys.exit(1)

            # Get the current configuration for defaults:
            configdir, configname = getconfignames()
            config = SaneConfigParser(configname)
            config.read(configname)

            # Create ESG content directory
            defaultRoot = config.get('DEFAULT', 'thredds_root')
            dn = os.path.dirname
            defaultTop = dn(dn(defaultRoot))
            threddsTop = askfor("TDS content directory", default=defaultTop)
            if not os.path.exists(threddsTop):
                tell("Directory %s does not exist. Please create the directory and rerun esgsetup."%threddsTop)
                raise RuntimeError
            threddsContent = os.path.join(threddsTop, "thredds")
            if not os.path.exists(threddsContent):
                tell("No content directory in %s"%threddsTop)
            threddsRoot = os.path.join(threddsTop, "thredds/esgcet")
            defaultThreddsTopURL = dn(config.get('DEFAULT', 'thredds_url'))
            urlfields = list(urlparse.urlparse(defaultThreddsTopURL))
            host = urlfields[1]
            hostfields = host.split()
            if len(hostfields)<3:
                hostname = socket.getfqdn()
                urlfields[1] = hostname
            defaultThreddsTopURL = urlparse.urlunparse(urlfields)
            threddsTopURL = askfor("URL of TDS root directory (this should include the full hostname or IP address, NOT localhost)", default=defaultThreddsTopURL)
            threddsURL = os.path.join(threddsTopURL, "esgcet")
            threddsMasterCatalog = askfor("Title of ESG master catalog (thredds_master_catalog_name)", default=config.get('DEFAULT', 'thredds_master_catalog_name', default="PCMDI Earth System Grid catalog"))
            threddsRootCatalog = askfor("Title of ESG root catalog (thredds_root_catalog_name)", default=config.get('DEFAULT', 'thredds_root_catalog_name', default="PCMDI Earth System Root catalog"))

            # Create ESG catalog directory
            if os.path.exists(threddsRoot):
                tell("Directory %s already exists"%threddsRoot)
            else:
                tell("Creating directory %s"%threddsRoot)
                os.makedirs(threddsRoot)

            # Set ESG configuration options
            updateconfig([('DEFAULT', 'thredds_root', threddsRoot),
                          ('DEFAULT', 'thredds_url', threddsURL),
                          ('DEFAULT', 'thredds_master_catalog_name', threddsMasterCatalog),
                          ('DEFAULT', 'thredds_root_catalog_name', threddsRootCatalog),
                          ], currentConfig=config)

            # Create initial ESG catalog
            from sqlalchemy import create_engine
            from sqlalchemy.orm import sessionmaker
            from esgcet.config import loadConfig, initLogging, getThreddsServiceSpecs
            from esgcet.publish import updateThreddsMasterCatalog, updateThreddsRootCatalog, reinitializeThredds
            from esgcet.exceptions import ESGPublishError

            config = loadConfig(configdir, load_projects=False)
            engine = create_engine(config.getdburl('extract'), echo=False, pool_recycle=3600)
            initLogging('DEFAULT', override_sa=engine)
            Session = sessionmaker(bind=engine, autoflush=True, autocommit=False)

            updateThreddsMasterCatalog(Session)

            # Add a link in the TDS catalog to the ESG catalog
            updateThreddsRootCatalog()

            # Test THREDDS reinit url
            threddsReinitURL = askfor("URL for TDS reinitialization [Note: the correct value differs between THREDDS V3 and V4] (thredds_reinit_url)", default=config.get('DEFAULT', 'thredds_reinit_url'))
            threddsErrorURL = askfor("URL of TDS error log [Note: the correct value differs between THREDDS V3 and V4] (thredds_reinit_error_url)", default=config.get('DEFAULT', 'thredds_reinit_error_url'))
            threddsUser = askfor("TDS username (thredds_username)", default=config.get('DEFAULT', 'thredds_username'))
            if threddsPassword is None:
                threddsPassword = askfor("TDS password", silent=True)
                threddsPassword2 = askfor("Confirm TDS password", silent=True)
                if threddsPassword!=threddsPassword2:
                    tell("Passwords do not match")
                    raise RuntimeError

            # Get the document roots
            currentRoots =  config.get('DEFAULT', 'thredds_dataset_roots')

            threddsDatasetRoots = askForMultiple("THREDDS dataset root directories (option=thredds_dataset_roots)\nEach entry has the form 'path_identifier | absolute_directory_path'", displayCurrent=currentRoots)

            # Get the gridFTP URL. If multiple file services are defined, create a compound service.
            threddsServiceSpecs = getThreddsServiceSpecs(config, 'DEFAULT', 'thredds_file_services')
            isCompound = (len(threddsServiceSpecs)>1)
            if isCompound:
                servTypes = [item[0] for item in threddsServiceSpecs]
                defaultCompoundName = DEFAULT_COMPOUND_FILE_SERVICE
                for servtype, servurl, servname, compoundName in threddsServiceSpecs:
                    if compoundName is not None:
                        defaultCompoundName = compoundName
                        break
                compoundName = askfor("Compound file service name (includes the services: %s)"%`servTypes`, default=defaultCompoundName)
            threddsFileSpecs = ''
            for servtype, servurl, servname, compName in threddsServiceSpecs:
                if servtype=='GridFTP':
                    servurl = askfor("THREDDS GridFTP service URL", default=servurl)
                    servname = askfor("THREDDS GridFTP service name", default=servname)
                if isCompound:
                    threddsFileSpecs += '\n%s | %s | %s | %s'%(servtype, servurl, servname, compoundName)
                else:
                    threddsFileSpecs += '\n%s | %s | %s'%(servtype, servurl, servname)

            # Set ESG configuration options
            updateconfig([('DEFAULT', 'thredds_reinit_url', threddsReinitURL),
                          ('DEFAULT', 'thredds_reinit_error_url', threddsErrorURL),
                          ('DEFAULT', 'thredds_username', threddsUser),
                          ('DEFAULT', 'thredds_password', threddsPassword),
                          ('DEFAULT', 'thredds_dataset_roots', threddsDatasetRoots),
                          ('DEFAULT', 'thredds_file_services', threddsFileSpecs),
                          ], currentConfig=config)
            

        except RuntimeError:
            tell("Script exited.")
            sys.exit(1)

    # remote publication initialization
    if initpublish:

        tell("""

        Setup the publication interface to the gateway:

        The publisher makes remote service calls to the index node. In this section you will set:

        - Service endpoints on the index node,
        - SSL port,
        - X509 certificate location.

        """)

        if not askyn("Proceed?"):
            raise RuntimeError

        try:

            # Get the current configuration for defaults:
            configdir, configname = getconfignames()
            config = SaneConfigParser(configname)
            config.read(configname)

            # Get defaults
            defaultGatewaySSLPort = config.get('DEFAULT', 'hessian_service_port', default="443")
            currentDebug = config.get('DEFAULT', 'hessian_service_debug', default="false")
            if currentDebug=="true":
                defaultHessianDebug = 'y'
            else:
                defaultHessianDebug = 'n'
            defaultProxyCertificate = config.get('DEFAULT', 'hessian_service_certfile', raw=True, default="%(home)s/.globus/certificate-file")

            # Get the service root
            svcroot = os.environ.get('ESG_GATEWAY_SVC_ROOT')
            if svcroot is None:
                svcroot = SERVICE_ROOT

            # Get the gateway host from ESG_GATEWAY_SVC_ROOT if set
            if gatewayHost is None:
                gatewayHost = urlparse.urlparse('http://'+svcroot).netloc

            # Get the default gateway URL from the remote metadata service URL
            defaultGatewayURL = "http://"+svcroot
            URLtuple = urlparse.urlparse(defaultGatewayURL)
            if gatewayHost is not None:
                defaultHessianServiceURL = urlparse.urlunparse(("http", gatewayHost, os.path.join(URLtuple[2], METADATA_SERVICE), '', '', ''))
            else:
                defaultHessianServiceURL = config.get('DEFAULT', 'hessian_service_remote_metadata_url', default=None)
            if defaultHessianServiceURL is not None:
                ind = defaultHessianServiceURL.find(METADATA_SERVICE)
                if ind!=-1:
                    defaultGatewayURL = defaultHessianServiceURL[:ind]

            gatewayURL = askfor("Gateway home URL", default=defaultGatewayURL)
            URLtuple = urlparse.urlparse(gatewayURL)
            hessianServiceURL = urlparse.urlunparse(("https", URLtuple[1], os.path.join(URLtuple[2], PUBLISH_SERVICE), '', '', ''))
            restServiceURL = urlparse.urlunparse(("https", URLtuple[1], os.path.join(URLtuple[2], REST_WEB_SERVICE), '', '', ''))
            solrSearchServiceURL = urlparse.urlunparse(("http", URLtuple[1], os.path.join(URLtuple[2], SOLR_SEARCH_SERVICE), '', '', ''))
            gatewaySSLPort = askfor("Gateway https port number", default=defaultGatewaySSLPort)
            proxyCertificate = askfor("Proxy certificate path (Hint: the '-o' argument to myproxy-logon)", default=defaultProxyCertificate)
            hessianDebug = askyn("Should protocol debugging be turned on? (This generates a lot of output)", default=defaultHessianDebug)
            if hessianDebug:
                hessianServiceDebug = "true"
            else:
                hessianServiceDebug = "false"

            # Set ESG configuration options
            updateconfig([('DEFAULT', 'hessian_service_url', hessianServiceURL),
                          ('DEFAULT', 'hessian_service_port', gatewaySSLPort),
                          ('DEFAULT', 'hessian_service_debug', hessianServiceDebug),
                          ('DEFAULT', 'hessian_service_certfile', proxyCertificate),
                          ('DEFAULT', 'hessian_service_keyfile', proxyCertificate),
                          ('DEFAULT', 'rest_service_url', restServiceURL),
                          ('DEFAULT', 'solr_search_service_url', solrSearchServiceURL),
                          ], currentConfig=config)

        except RuntimeError:
            tell("Script exited.")
            sys.exit(1)

    # Customized handler setup
    if generateHandler:

        tell("""

        Setup a custom handler package.

        There are four types of handlers Any or all types of handler can
        be included:

        - Project handlers encapsulate the logic about what metadata should be associated
          with a project, and how it is represented in the data, directory structures, etc.
        - Format handlers describe the I/O for a specific file format such as netCDF.
        - Metadata handlers describe the metadata conventions and time value representation
          associated with a specific metadata convention, such as the CF convention.
        - A THREDDS catalog hook is a function for customizing the TDS catalog files.

        This section will create a skeleton handler package for one or more handler types.
        You will have the option to select:

        - The package name and directory;
        - Which type of handler(s) to create, and the name of each handler
        - For each handler, the name that will identify it in the esg.ini 'handler_name' option.

        """)
        if not askyn("Proceed?"):
            raise RuntimeError

        dirName = os.path.expanduser(askfor("In what directory should the handler package be created?"))
        packageName = askfor("Package name (no spaces or special characters)")
        packageDir = os.path.expanduser(os.path.join(dirName, packageName))
        packageDescription = askfor("Package description")
        author = askfor("Author")
        setupProject = askyn("Set up a project handler?", default='n')
        setupFormat = askyn("Set up a format handler?", default='n')
        setupMetadata = askyn("Set up a metadata handler?", default='n')
        setupThredds = askyn("Set up a THREDDS catalog hook?", default='n')

        if not (setupProject or setupFormat or setupMetadata or setupThredds):
            print 'No handlers initialized.'
            sys.exit(0)

        # Create the directory if necessary
        if not os.path.exists(dirName):
            print 'Creating %s'%dirName
            os.makedirs(dirName)
        if not os.path.exists(packageDir):
            print 'Creating %s'%packageDir
            os.makedirs(packageDir)

        handlerList = ""
        handlerEntryPoints = ""
        optionList = ""

        # Create the handler skeleton files
        from esgcet.config import ESGCET_PROJECT_HANDLER_GROUP, ESGCET_FORMAT_HANDLER_GROUP, ESGCET_METADATA_HANDLER_GROUP, ESGCET_THREDDS_CATALOG_HOOK_GROUP
        if setupProject:
            handlerList, handlerEntryPoints, optionList = setupHandler(packageDir, packageName, 'project', handlerList, handlerEntryPoints, optionList, ESGCET_PROJECT_HANDLER_GROUP, 'CustomProjectHandler')
        if setupFormat:
            handlerList, handlerEntryPoints, optionList = setupHandler(packageDir, packageName, 'format', handlerList, handlerEntryPoints, optionList, ESGCET_FORMAT_HANDLER_GROUP, 'CustomFormatHandler')
        if setupMetadata:
            handlerList, handlerEntryPoints, optionList = setupHandler(packageDir, packageName, 'metadata', handlerList, handlerEntryPoints, optionList, ESGCET_METADATA_HANDLER_GROUP, 'CustomMetadataHandler')
        if setupThredds:
            handlerClass = 'threddsCatalogHook'
            handlerList, handlerEntryPoints, optionList = setupHandler(packageDir, packageName, None, handlerList, handlerEntryPoints, optionList, ESGCET_THREDDS_CATALOG_HOOK_GROUP, handlerClass, handler="%s.thredds_catalog_hook:%s"%(packageName, handlerClass), handlerBasename="thredds_catalog_hook.py", handlerTemplate="thredds_catalog_hook.py.tmpl", handlerOption="thredds_catalog_hook")

        # Create setup.py
        from pkg_resources import resource_string
        setupPath = os.path.join(dirName, 'setup.py')
        createit = True
        if os.path.exists(setupPath):
            createit = askyn('Replace existing file %s?'%setupPath)
        if createit:
            setupString = resource_string('esgcet.config.etc', 'handler_setup.py.tmpl')
            setupString = setupString%(author, packageName, packageDescription, packageName, handlerEntryPoints)
            f = open(setupPath, 'w')
            f.write(setupString)
            f.close()

        # Create an empty __init__.py
        initPath = os.path.join(packageDir, '__init__.py')
        createit = True
        if os.path.exists(initPath):
            createit = askyn('Replace existing file %s?'%initPath)
        if createit:
            f = open(initPath, 'w')
            f.close()

        # Next steps
        tell("""
        A skeleton handler package has been created in %s.
        To complete the handler setup:

        (1) Edit the following handler files:
        %s

        (2) Install the handler package with:

        %% cd %s
        %% python setup.py --verbose install

        (3) Set the following options in esg.ini, in the appropriate project section(s):
        %s

        """%(dirName, handlerList, dirName, optionList))

if __name__=='__main__':
    main(sys.argv[1:])
