#!/usr/bin/env python3
# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
#  - Cedric Serfon, <cedric.serfon@cern.ch>, 2018
#  - Jaroslav Guenther, <jaroslav.guenther@cern.ch>, 2019
#  - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
# PY3K COMPATIBLE

"""
Replica-Recoverer is a daemon that declares suspicious replicas as bad if they are found available on other RSE.
Consequently, automatic replica recovery is triggered via necromancer daemon, which creates a rule for such bad replicas.
"""

import argparse
import signal

from rucio.daemons.replicarecoverer.suspicious_replica_recoverer import run, stop


def get_parser():
    """
    Returns the argparse parser.
    """
    parser = argparse.ArgumentParser(description='''

Replica-Recoverer is a daemon that declares suspicious replicas
that are available on other RSE as bad. Consequently, automatic
replica recovery is triggered via necromancer daemon,
which creates a rule for such bad replicas.''', epilog='''

Preparing RSEs and DIDs for testing this daemon from rucio docker:
-------------------------------------------------------------------

$ sudo docker exec -it dev_rucio_1 /bin/bash

Adding the RSEs to rucio DB:

$ rucio-admin rse add MOCK_SUSPICIOUS
$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key backend_type --value file
$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse1/' --space-token 'ATLASDATADISK1' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_SUSPICIOUS
$ rucio-admin rse info MOCK_SUSPICIOUS

$ rucio-admin rse add MOCK_RECOVERY
$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key backend_type --value POSIX
$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse2/' --space-token 'ATLASDATADISK2' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_RECOVERY
$ rucio-admin rse info MOCK_RECOVERY

For testing, we create the following files:
-------------------------------------------------------------------

Name                                  MOCK_RECOVERY       MOCK_SUSPICIOUS    BAD (on MOCK_SUSPICIOUS)    SUSPICIOUS (on MOCK_SUSPICIOUS)
file_available_suspicious             yes                 yes                no                          yes
file_available_suspicious_and_bad     yes                 yes                yes                         yes
file_notavailable_suspicious          unavailable         yes                no                          yes

Only file_available_suspicious should be the one on which the daemon takes action and declares it as bad.

$ id0=`uuidgen`
$ id1=`uuidgen`
$ id2=`uuidgen`
$ id3=`uuidgen`
$ echo "file available on MOCK_RECOVERY and decalred suspicious on MOCK_SUSPICIOUS  (11 times)" > /tmp/file_available_suspicious'_'$id1
$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS  (11 times) and 1 time bad/deleted/lost on MOCK_SUSPICIOUS" > /tmp/file_available_suspicious_and_bad'_'$id2
$ echo "file declared as unavailable on MOCK_RECOVERY and declared as suspicious 11 times on MOCK_SUSPICIOUS" > /tmp/file_notavailable_suspicious'_'$id3

Uploading the files created above to rucio:
-------------------------------------------------------------------

rucio add-dataset mock:dataset_of_suspicious_replicas'_'$id0

Added mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778

$ rucio add-rule mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_SUSPICIOUS
$ rucio add-rule --source-replica-expression MOCK_SUSPICIOUS mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_RECOVERY
$ rucio list-rules mock:dataset_of_suspicious_replicas'_'$id0

ID                                ACCOUNT    SCOPE:NAME                                                                STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
--------------------------------  ---------  ------------------------------------------------------------------------  ----------------------  ----------------  --------  ---------------  -------------------
2a1a078b66ca4e209cc20a5826125334  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_RECOVERY            1                   2019-02-19 14:12:30
8c15d2f8e94a459a86a488055a10d068  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_SUSPICIOUS          1                   2019-02-19 14:12:28

$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
[...]
$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious'_'$id1
$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious_and_bad'_'$id2
$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_notavailable_suspicious'_'$id3
[...]

$ rucio list-file-replicas mock:file_available_suspicious'_'$id1
+---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+
| SCOPE   | NAME                                                           | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                 |
|---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------|
| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e |
| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e   |
+---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+

$ rucio list-file-replicas mock:file_available_suspicious_and_bad'_'$id2
+---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+
| SCOPE   | NAME                                                                   | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                         |
|---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------|
| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 |
| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835   |
+---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+

$ rucio list-file-replicas mock:file_notavailable_suspicious'_'$id3
+---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+
| SCOPE   | NAME                                                              | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                    |
|---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------|
| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f |
| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f   |
+---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+

Modifying the file statuses in the DB:
--------------------------------------

$ python

# the paths below point to MOCK_SUSPICIOUS RSE (.../rucio_rse1)

$$ file1 = ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e',]
$$ file2 = ['file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835',]
$$ file3 = ['file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f' ]

$$ from rucio.client.replicaclient import ReplicaClient
$$ replica_client = ReplicaClient()
$$ import time
$$ for i in range(11):
       replica_client.declare_suspicious_file_replicas(file1, 'This is a good reason')
       replica_client.declare_suspicious_file_replicas(file2, 'This is a good reason')
       replica_client.declare_suspicious_file_replicas(file3, 'This is a good reason')
       time.sleep(1)

#  Declaring file2 bad on MOCK_SUSPICIOUS:
   ---------------------------------------

$$ replica_client.declare_bad_file_replicas(file2, 'This is a good reason')

#  Update replica state of 'file_notavailable_suspicious'_'$id1' on MOCK_RECOVERY to 'UNAVAILABLE'
#  (change the file name below according to the info from rucio !):
   -----------------------------------------------------------------------------------------------

$$ replica_client.update_replicas_states('MOCK_RECOVERY', [{'scope':'mock', 'name':'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'state':'U'}])


# Checking the results of the file status changes:
  ------------------------------------------------
$$ from rucio.core.replica import get_suspicious_files
$$ from datetime import datetime, timedelta
$$ from_date = datetime.now() - timedelta(days=3)
$$ from rucio.core.replica import list_bad_replicas_status

$$ get_suspicious_files('MOCK_SUSPICIOUS',from_date,10)

[{'created_at': datetime.datetime(2019, 2, 19, 14, 12, 56), 'scope': u'mock', 'cnt': 11L, 'name': u'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'rse': u'MOCK_SUSPICIOUS'},
 {'created_at': datetime.datetime(2019, 2, 19, 14, 12, 48), 'scope': u'mock', 'cnt': 11L, 'name': u'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': u'MOCK_SUSPICIOUS'}]

$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)

[{'name': u'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': u'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': u'mock'}]

$$ exit()

Run the daemon:
---------------

  $ python bin/rucio-replica-recoverer --run-once --rse-expression='MOCK_SUSPICIOUS'

Terminal output:
----------------

2019-02-19 14:39:24,114 709 INFO  replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
2019-02-19 14:39:24,124 709 INFO  replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': u'mock', 'cnt': 11L, 'name': u'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': u'MOCK_SUSPICIOUS'}]
2019-02-19 14:39:24,125 709 INFO  replica_recoverer[0/0]: looking for replica surls.
2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: found 1/1 surls (took 0.035572052002 seconds) - declaring them as bad replicas now.
2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: [u'file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
2019-02-19 14:39:24,188 709 INFO  replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
2019-02-19 14:39:24,192 709 INFO  replica_recoverer[0/0]: graceful stop done


# Checking the results of the file status changes:
  ------------------------------------------------
$ python

$$ from rucio.core.replica import get_suspicious_files
$$ from datetime import datetime, timedelta
$$ from_date = datetime.now() - timedelta(days=3)
$$ from rucio.core.replica import list_bad_replicas_status

$$ get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=True)

>>> get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=True)
[]

$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)

[{'name': u'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': u'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'updated_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'state': BAD, 'scope': u'mock'},
 {'name': u'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': u'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': u'mock'}]

$$ exit()

When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::

  $ rucio-replica-recoverer --run-once
  2020-07-28 15:15:14,151 5461    INFO    replica_recoverer: This instance will work on VOs: def, abc, xyz, 123

By using the ``--vos`` argument only the VO or VOs specified will be affected::

  $ rucio-replica-recoverer --run-once --vos abc xyz
  2020-07-28 15:16:36,066 5474    INFO    replica_recoverer: This instance will work on VOs: abc, xyz

Note that attempting the use the ``--vos`` argument when in single-VO mode will have no affect::

  $ rucio-replica-recoverer --run-once --vos abc xyz
  2020-07-28 15:21:33,349 5488    WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
''')  # NOQA: E501
    parser.add_argument("--nattempts", action="store", default=10, help='Minimum count of suspicious file replica appearance in bad_replicas table.Default value is 10.')
    parser.add_argument("--younger-than", action="store", default=3, help='Consider all file replicas logged in bad_replicas table since speicified number of younger-than. Default value is 3.')
    parser.add_argument("--rse-expression", action="store", default='MOCK', help='The RSE on which the recovery should be happening.')
    parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
    parser.add_argument("--max-replicas-per-rse", action="store", default=100, help='The maximum number of suspicious replicas found on an RSE which may be declared bad. If a higher count is found, no action is be taken.')
    return parser


if __name__ == "__main__":
    signal.signal(signal.SIGTERM, stop)
    PARSER = get_parser()
    ARGS = PARSER.parse_args()
    try:
        run(once=ARGS.run_once, younger_than=ARGS.younger_than, nattempts=ARGS.nattempts, rse_expression=ARGS.rse_expression, vos=ARGS.vos, max_replicas_per_rse=ARGS.max_replicas_per_rse)
    except KeyboardInterrupt:
        stop()
