"""
Encapsulate the different transports available to Salt.

This includes server side transport, for the ReqServer and the Publisher
"""

import asyncio
import binascii
import collections
import errno
import hashlib
import hmac
import logging
import os
import pathlib
import random
import string
import time
import zlib

import tornado.ioloop

import salt.cache
import salt.crypt
import salt.master
import salt.payload
import salt.transport
import salt.transport.frame
import salt.transport.tcp
import salt.utils.channel
import salt.utils.event
import salt.utils.minions
import salt.utils.platform
import salt.utils.stringutils
from salt.exceptions import SaltDeserializationError, UnsupportedAlgorithm
from salt.utils.cache import CacheCli

log = logging.getLogger(__name__)


def _get_crypticle(opts, key_string, key_size=192, serial=0):
    """
    Get appropriate Crypticle class based on configuration.

    Returns TLSAwareCrypticle if TLS optimization is enabled, otherwise
    returns standard Crypticle.

    Args:
        opts: Configuration dictionary
        key_string: AES key string
        key_size: Key size in bits (default: 192)
        serial: Serial number (default: 0)

    Returns:
        Crypticle or TLSAwareCrypticle instance
    """
    if opts.get("disable_aes_with_tls", False):
        return salt.crypt.TLSAwareCrypticle(opts, key_string, key_size, serial)
    else:
        return salt.crypt.Crypticle(opts, key_string, key_size, serial)


def cluster_pub_matches_fingerprint(opts, cluster_pub):
    """
    Verify a received cluster public key against a pinned fingerprint.

    When ``opts["cluster_pub_fingerprint"]`` is set, the joining master
    requires the ``cluster_pub`` it receives in a
    ``cluster/peer/discover-reply`` to hash to that value (SHA-256 hex
    digest of the PEM bytes, case-insensitive). When the option is unset
    this function returns ``True`` unconditionally, which is the
    trust-on-first-contact behavior documented for deployments that share
    ``cluster_pki_dir`` over a filesystem.

    ``cluster_pub`` may be a ``str`` (PEM text) or ``bytes``.

    Returns ``True`` on match (or when no fingerprint is pinned) and
    ``False`` on mismatch.
    """
    pinned = opts.get("cluster_pub_fingerprint")
    if not pinned:
        return True
    if isinstance(cluster_pub, str):
        pub_bytes = cluster_pub.encode()
    else:
        pub_bytes = cluster_pub
    digest = hashlib.sha256(pub_bytes).hexdigest()
    return hmac.compare_digest(digest.lower(), str(pinned).lower())


class ReqServerChannel:
    """
    ReqServerChannel handles request/reply messages from ReqChannels.
    """

    @classmethod
    def factory(cls, opts, **kwargs):
        """
        Return the appropriate server channel for the configured transport.

        Two mutually exclusive code paths exist, selected here at startup:

        1. **Pooled** (``worker_pools_enabled=True``, the default):
           Returns a :class:`PoolRoutingChannel` that sits in front of the
           external transport.  Incoming requests are routed to per-pool IPC
           RequestServers and dispatched to MWorkers.  Clear-text ``_auth``
           uses that IPC path when connected; before IPC clients exist it is
           handled inline (same semantics as the non-pooled channel).

        2. **Non-pooled** (``worker_pools_enabled=False``, legacy):
           Returns a plain :class:`ReqServerChannel` whose
           :meth:`handle_message` intercepts ``_auth`` inline (before the
           payload ever reaches a worker) and handles it directly via
           :meth:`_auth`.  All other commands are forwarded to the single
           worker pool via ``payload_handler``.

        These paths are mutually exclusive at runtime; ``_auth`` is not run
        twice for a single request.
        """
        if "master_uri" not in opts and "master_uri" in kwargs:
            opts["master_uri"] = kwargs["master_uri"]

        # Handle worker pool routing if enabled.
        # PoolRoutingChannel is now the default implementation when
        # worker_pools_enabled=True. We only wrap if we are NOT already a
        # pool-specific server (to avoid recursion).
        if opts.get("worker_pools_enabled", True) and not opts.get("pool_name"):
            from salt.config.worker_pools import get_worker_pools_config

            worker_pools = get_worker_pools_config(opts)
            if worker_pools:
                # Wrap the standard transport in the routing channel
                external_opts = opts.copy()
                external_opts["worker_pools_enabled"] = False
                import salt.transport.base

                transport = salt.transport.base.request_server(external_opts, **kwargs)
                return PoolRoutingChannel(opts, transport, worker_pools)

        import salt.transport.base

        transport = salt.transport.base.request_server(opts, **kwargs)
        return cls(opts, transport)

    @classmethod
    def compare_keys(cls, key1, key2):
        """
        Normalize and compare two keys

        Returns:
            bool: ``True`` if the keys match, otherwise ``False``
        """
        return salt.crypt.clean_key(key1) == salt.crypt.clean_key(key2)

    def __init__(self, opts, transport):
        self.opts = opts
        self.transport = transport
        self.cache = salt.cache.Cache(opts, driver=self.opts["keys.cache_driver"])
        self.event = salt.utils.event.get_master_event(
            self.opts, self.opts["sock_dir"], listen=False
        )
        self.master_key = salt.crypt.MasterKeys(self.opts)

        (pathlib.Path(self.opts["cachedir"]) / "sessions").mkdir(exist_ok=True)
        self.sessions = {}

    @property
    def aes_key(self):
        if self.opts.get("cluster_id", None):
            return salt.master.SMaster.secrets["cluster_aes"]["secret"].value
        return salt.master.SMaster.secrets["aes"]["secret"].value

    def session_key(self, minion):
        """
        Returns a session key for the given minion id.
        """
        now = time.time()
        if minion in self.sessions:
            if now - self.sessions[minion][0] < self.opts["publish_session"]:
                return self.sessions[minion][1]

        path = pathlib.Path(self.opts["cachedir"]) / "sessions" / minion
        try:
            if now - path.stat().st_mtime > self.opts["publish_session"]:
                salt.crypt.Crypticle.write_key(path)
        except FileNotFoundError:
            salt.crypt.Crypticle.write_key(path)

        self.sessions[minion] = (
            path.stat().st_mtime,
            salt.crypt.Crypticle.read_key(path),
        )
        return self.sessions[minion][1]

    def pre_fork(self, process_manager, *args, **kwargs):
        """
        Do anything necessary pre-fork. Since this is on the master side this will
        primarily be bind and listen (or the equivalent for your network library)
        """
        import salt.master

        if "secrets" not in kwargs:
            kwargs["secrets"] = salt.master.SMaster.secrets
        if hasattr(self.transport, "pre_fork"):
            self.transport.pre_fork(process_manager, *args, **kwargs)

    def post_fork(self, payload_handler, io_loop, **kwargs):
        """
        Do anything you need post-fork. This should handle all incoming payloads
        and call payload_handler. You will also be passed io_loop, for all of your
        asynchronous needs
        """
        import salt.master

        if self.opts["pub_server_niceness"] and not salt.utils.platform.is_windows():
            log.info(
                "setting Publish daemon niceness to %i",
                self.opts["pub_server_niceness"],
            )
            os.nice(self.opts["pub_server_niceness"])
        self.io_loop = io_loop
        self.crypticle = _get_crypticle(self.opts, self.aes_key)
        # other things needed for _auth
        # Create the event manager
        self.event = salt.utils.event.get_master_event(
            self.opts, self.opts["sock_dir"], listen=False, io_loop=io_loop
        )
        self.auto_key = salt.daemons.masterapi.AutoKey(self.opts)
        # only create a con_cache-client if the con_cache is active
        if self.opts["con_cache"]:
            self.cache_cli = CacheCli(self.opts)
        else:
            self.cache_cli = False
            # Make an minion checker object
            self.ckminions = salt.utils.minions.CkMinions(self.opts)
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.payload_handler = payload_handler
        if hasattr(self.transport, "post_fork"):
            self.transport.post_fork(self.handle_message, io_loop, **kwargs)

    async def handle_message(self, payload):
        """
        Handle an incoming request payload (non-pooled / legacy path only).

        This method is only active when ``worker_pools_enabled=False``.  In
        that configuration this channel owns the external transport socket and
        processes every request inline.

        ``_auth`` handling
        ------------------
        When the payload command is ``_auth`` this method calls
        :meth:`_auth` directly and returns the result without forwarding the
        payload to any worker.  This is the **only** place ``_auth`` executes
        in the non-pooled path.

        All other commands are forwarded to a worker via ``payload_handler``
        (i.e. :meth:`~salt.master.MWorker._handle_payload`).

        See :meth:`factory` for the full description of the two mutually
        exclusive request paths and why ``_auth`` is always executed exactly
        once.
        """
        nonce = None
        if (
            not isinstance(payload, dict)
            or "enc" not in payload
            or "load" not in payload
        ):
            log.warning("bad load received on socket")
            return "bad load"
        try:
            version = int(payload.get("version", 0))
        except ValueError:
            version = 0

        # Enforce minimum authentication protocol version to prevent downgrade attacks
        minimum_version = self.opts.get("minimum_auth_version", 0)
        if minimum_version > 0 and version < minimum_version:
            load = payload.get("load")
            if isinstance(load, dict):
                minion_id = load.get("id", "unknown minion")
            else:
                minion_id = "unknown minion"
            log.warning(
                "Rejected authentication attempt from minion '%s' using "
                "protocol version %d (minimum required: %d)",
                minion_id,
                version,
                minimum_version,
            )
            return "bad load"

        try:
            payload = self._decode_payload(payload, version)
        except Exception as exc:  # pylint: disable=broad-except
            exc_type = type(exc).__name__
            if exc_type == "AuthenticationError":
                log.debug(
                    "Minion failed to auth to master. Since the payload is "
                    "encrypted, it is not known which minion failed to "
                    "authenticate. It is likely that this is a transient "
                    "failure due to the master rotating its public key."
                )
            else:
                log.error("Bad load from minion: %s: %s", exc_type, exc)
            return "bad load"

        # TODO helper functions to normalize payload?
        if not isinstance(payload, dict) or not isinstance(payload.get("load"), dict):
            log.error(
                "payload and load must be a dict. Payload was: %s",
                payload,
            )
            return "payload and load must be a dict"

        try:
            id_ = payload["load"].get("id", "")
            if "\0" in id_:
                log.error("Payload contains an id with a null byte: %s", payload)
                return "bad load: id contains a null byte"
        except TypeError:
            log.error("Payload contains non-string id: %s", payload)
            return f"bad load: id {id_} is not a string"

        sign_messages = False
        if version > 1:
            sign_messages = True

        if payload["enc"] == "aes":
            nonce = None
            if version > 1:
                nonce = payload["load"].pop("nonce", None)

            # Check validity of message ttl and id's match
            if version > 2:
                if self.opts["request_server_ttl"] > 0:
                    ttl = time.time() - payload["load"]["ts"]
                    if ttl > self.opts["request_server_ttl"]:
                        log.warning(
                            "Received request from %s with expired ttl: %d > %d",
                            payload["load"]["id"],
                            ttl,
                            self.opts["request_server_ttl"],
                        )
                        return "bad load"

                if payload["id"] != payload["load"]["id"]:
                    log.warning(
                        "Request id mismatch. Found '%s' but expected '%s'",
                        payload["load"]["id"],
                        payload["id"],
                    )
                    return "bad load"
                if not salt.utils.verify.valid_id(self.opts, payload["load"]["id"]):
                    log.warning(
                        "Request contains invalid minion id '%s'", payload["load"]["id"]
                    )
                    return "bad load"
                if not self.validate_token(payload, required=True):
                    return "bad load"
            # The token won't always be present in the payload for and
            # below, but if it is we always wanto validate it.
            elif not self.validate_token(payload, required=False):
                return "bad load"

        # TODO: test
        try:
            # intercept the "_auth" commands, since the main daemon shouldn't know
            # anything about our key auth
            if (
                payload["enc"] == "clear"
                and payload.get("load", {}).get("cmd") == "_auth"
            ):
                # Store time at the beginning of serving _auth call
                # to calculate duration of the call with master_stats
                start = time.time()
                ret = self._auth(payload["load"], sign_messages, version)
                if self.opts.get("master_stats", False):
                    await self.payload_handler({"cmd": "_auth", "_start": start})
                return ret

            # Take the payload_handler function that was registered when we created the channel
            # and call it, returning control to the caller until it completes

            ret, req_opts = await self.payload_handler(payload)

            req_fun = req_opts.get("fun", "send")
            if req_fun == "send_clear":
                return ret
            elif req_fun == "send":
                if version > 2:
                    return _get_crypticle(self.opts, self.session_key(id_)).dumps(
                        ret, nonce
                    )
                else:
                    return self.crypticle.dumps(ret, nonce)
            elif req_fun == "send_private":
                return self._encrypt_private(
                    ret,
                    req_opts["key"],
                    req_opts["tgt"],
                    nonce,
                    sign_messages,
                    payload.get("enc_algo", salt.crypt.OAEP_SHA1),
                    payload.get("sig_algo", salt.crypt.PKCS1v15_SHA1),
                )
            log.error("Unknown req_fun %s", req_fun)
            # always attempt to return an error to the minion
            return "Server-side exception handling payload"

        except Exception as e:  # pylint: disable=broad-except
            # always attempt to return an error to the minion
            log.error("Some exception handling a payload from minion", exc_info=True)
            return "Some exception handling minion payload"

    def _encrypt_private(
        self,
        ret,
        dictkey,
        target,
        nonce=None,
        sign_messages=True,
        encryption_algorithm=salt.crypt.OAEP_SHA1,
        signing_algorithm=salt.crypt.PKCS1v15_SHA1,
    ):
        """
        The server equivalent of ReqChannel.crypted_transfer_decode_dictentry
        """
        # encrypt with a specific AES key
        try:
            key = salt.crypt.Crypticle.generate_key_string()
            pcrypt = _get_crypticle(self.opts, key)
            pub = self.cache.fetch("keys", target)
            if not isinstance(pub, dict) or "pub" not in pub:
                log.error(
                    "No pub key found for target %s, its pub key was likely deleted mid-request.",
                    target,
                )
                return self.crypticle.dumps({})

            pub = salt.crypt.PublicKey.from_str(pub["pub"])
        except Exception as exc:  # pylint: disable=broad-except
            log.error(
                'Corrupt or missing public key "%s": %s',
                target,
                exc,
                exc_info_on_loglevel=logging.DEBUG,
            )
            return self.crypticle.dumps({})
        pret = {}
        pret["key"] = pub.encrypt(key, encryption_algorithm)
        if ret is False:
            ret = {}
        if sign_messages:
            if nonce is None:
                return {"error": "Nonce not included in request"}
            tosign = salt.payload.dumps(
                {"key": pret["key"], "pillar": ret, "nonce": nonce}
            )
            signed_msg = {
                "data": tosign,
                "sig": self.master_key.sign(tosign, algorithm=signing_algorithm),
            }
            pret[dictkey] = pcrypt.dumps(signed_msg)
        else:
            pret[dictkey] = pcrypt.dumps(ret)
        return pret

    def _clear_signed(self, load, algorithm):
        try:
            tosign = salt.payload.dumps(load)
            return {
                "enc": "clear",
                "load": tosign,
                "sig": self.master_key.sign(tosign, algorithm=algorithm),
            }
        except UnsupportedAlgorithm:
            log.info(
                "Minion tried to authenticate with unsupported signing algorithm: %s",
                algorithm,
            )
            return {"enc": "clear", "load": {"ret": "bad sig algo"}}

    def _update_aes(self):
        """
        Check to see if a fresh AES key is available and update the components
        of the worker
        """
        import salt.master

        key = "aes"
        if self.opts.get("cluster_id", None):
            key = "cluster_aes"

        if (
            salt.master.SMaster.secrets[key]["secret"].value
            != self.crypticle.key_string
        ):
            self.crypticle = _get_crypticle(
                self.opts, salt.master.SMaster.secrets[key]["secret"].value
            )
            return True
        return False

    def _decode_payload(self, payload, version):
        # we need to decrypt it
        if payload["enc"] == "aes":
            if version > 2:
                if salt.utils.verify.valid_id(self.opts, payload["id"]):
                    payload["load"] = _get_crypticle(
                        self.opts,
                        self.session_key(payload["id"]),
                    ).loads(payload["load"])
                else:
                    raise SaltDeserializationError("Encountered invalid id")
            else:
                try:
                    payload["load"] = self.crypticle.loads(payload["load"])
                except salt.crypt.AuthenticationError:
                    if not self._update_aes():
                        raise
                    payload["load"] = self.crypticle.loads(payload["load"])
        return payload

    def validate_token(self, payload, required=True):
        """
        Validate the token (tok) and minion id (id) in the payload. If the
        payload and token exist they will be validated even if required is
        False.

        When required is False and either the tok or id is not found in the
        load, this check will pass.

        This method has a side effect of removing the 'tok' key from the load
        so that it is not passed along to request handlers.
        """
        tok = payload["load"].pop("tok", None)
        id_ = payload["load"].get("id", None)
        if tok is not None and id_ is not None:
            if "cluster_id" in self.opts and self.opts["cluster_id"]:
                pki_dir = self.opts["cluster_pki_dir"]
            else:
                pki_dir = self.opts.get("pki_dir", "")
            try:
                pub_path = salt.utils.verify.clean_join(pki_dir, "minions", id_)
            except salt.exceptions.SaltValidationError:
                log.warning("Invalid minion id: %s", id_)
                return False
            try:
                pub = salt.crypt.PublicKey.from_file(pub_path)
            except OSError:
                log.warning(
                    "Salt minion claiming to be %s attempted to communicate with "
                    "master, but key could not be read and verification was denied.",
                    id_,
                )
                return False
            try:
                if pub.decrypt(tok) != b"salt":
                    log.error("Minion token did not validate: %s", id_)
                    return False
            except ValueError as err:
                log.error("Unable to decrypt token: %s", err)
                return False
        elif required:
            return False
        return True

    def _auth(self, load, sign_messages=False, version=0):
        """
        Authenticate the client, use the sent public key to encrypt the AES key
        which was generated at start up.

        This method fires an event over the master event manager. The event is
        tagged "auth" and returns a dict with information about the auth
        event

            - Verify that the key we are receiving matches the stored key
            - Store the key if it is not there
            - Make an RSA key with the pub key
            - Encrypt the AES key as an encrypted salt.payload
            - Package the return and return it
        """
        import salt.master

        enc_algo = load.get("enc_algo", salt.crypt.OAEP_SHA1)
        sig_algo = load.get("sig_algo", salt.crypt.PKCS1v15_SHA1)

        if not salt.utils.verify.valid_id(self.opts, load["id"]):
            log.info("Authentication request from invalid id %s", load["id"])
            if sign_messages:
                return self._clear_signed(
                    {"ret": False, "nonce": load["nonce"]}, sig_algo
                )
            else:
                return {"enc": "clear", "load": {"ret": False}}
        log.info("Authentication request from %s", load["id"])
        # remove any trailing whitespace
        load["pub"] = load["pub"].strip()

        # 0 is default which should be 'unlimited'
        if self.opts["max_minions"] > 0:
            # use the ConCache if enabled, else use the minion utils
            if self.cache_cli:
                minions = self.cache_cli.get_cached()
            else:
                minions = self.ckminions.connected_ids()
                if len(minions) > 1000:
                    log.info(
                        "With large numbers of minions it is advised "
                        "to enable the ConCache with 'con_cache: True' "
                        "in the masters configuration file."
                    )

            if not len(minions) <= self.opts["max_minions"]:
                # we reject new minions, minions that are already
                # connected must be allowed for the mine, highstate, etc.
                if load["id"] not in minions:
                    log.info(
                        "Too many minions connected (max_minions=%s). "
                        "Rejecting connection from id %s",
                        self.opts["max_minions"],
                        load["id"],
                    )

                    if self.opts.get("auth_events") is True:
                        eload = {
                            "result": False,
                            "act": "full",
                            "id": load["id"],
                            "pub": load["pub"],
                        }
                        autosign_grains = load.get("autosign_grains", None)
                        if (
                            "full" in self.opts.get("auth_events_autosign_grains", [])
                            and autosign_grains
                        ):
                            eload["autosign_grains"] = autosign_grains
                        self.event.fire_event(
                            eload, salt.utils.event.tagify(prefix="auth")
                        )
                    if sign_messages:
                        return self._clear_signed(
                            {"ret": "full", "nonce": load["nonce"]}, sig_algo
                        )
                    else:
                        return {"enc": "clear", "load": {"ret": "full"}}

        # Check if key is configured to be auto-rejected/signed
        auto_reject = self.auto_key.check_autoreject(load["id"])
        auto_sign = self.auto_key.check_autosign(
            load["id"], load.get("autosign_grains", None)
        )

        # key will be a dict of str and state
        # state can be one of pending, rejected, accepted
        key = self.cache.fetch("keys", load["id"])

        # although keys should be always newline stripped in current state of auth.py
        # older salt versions  may have written pub-keys with trailing whitespace
        if key and "pub" in key:
            key["pub"] = key["pub"].strip()

        # any number of keys can be denied for a given minion_id regardless of above
        denied = self.cache.fetch("denied_keys", load["id"]) or []

        if self.opts["open_mode"]:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif key and key["state"] == "rejected":
            # The key has been rejected, don't place it in pending
            log.info(
                "Public key rejected for %s. Key is present in rejection key dir.",
                load["id"],
            )
            if self.opts.get("auth_events") is True:
                eload = {
                    "result": False,
                    "act": "reject",
                    "id": load["id"],
                    "pub": load["pub"],
                }
                autosign_grains = load.get("autosign_grains", None)
                if (
                    "reject" in self.opts.get("auth_events_autosign_grains", [])
                    and autosign_grains
                ):
                    eload["autosign_grains"] = autosign_grains
                self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
            if sign_messages:
                return self._clear_signed(
                    {"ret": False, "nonce": load["nonce"]}, sig_algo
                )
            else:
                return {"enc": "clear", "load": {"ret": False}}
        elif key and key["state"] == "accepted":
            # The key has been accepted, check it
            if not self.compare_keys(key["pub"], load["pub"]):
                log.error(
                    "Authentication attempt from %s failed, the public "
                    "keys did not match. This may be an attempt to compromise "
                    "the Salt cluster.",
                    load["id"],
                )
                # put denied minion key into minions_denied
                if load["pub"] not in denied:
                    denied.append(load["pub"])
                    self.cache.store("denied_keys", load["id"], denied)

                if self.opts.get("auth_events") is True:
                    eload = {
                        "result": False,
                        "id": load["id"],
                        "act": "denied",
                        "pub": load["pub"],
                    }
                    autosign_grains = load.get("autosign_grains", None)
                    if (
                        "denied" in self.opts.get("auth_events_autosign_grains", [])
                        and autosign_grains
                    ):
                        eload["autosign_grains"] = autosign_grains
                    self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
                if sign_messages:
                    return self._clear_signed(
                        {"ret": False, "nonce": load["nonce"]}, sig_algo
                    )
                else:
                    return {"enc": "clear", "load": {"ret": False}}

        elif not key:
            # The key has not been accepted, this is a new minion
            key_act = None
            if auto_reject:
                log.info(
                    "New public key for %s rejected via autoreject_file", load["id"]
                )
                key = {"pub": load["pub"], "state": "rejected"}
                self.cache.store("keys", load["id"], key)
                key_act = "reject"
                key_result = False
            elif not auto_sign:
                log.info("New public key for %s placed in pending", load["id"])
                key = {"pub": load["pub"], "state": "pending"}
                self.cache.store("keys", load["id"], key)
                key_act = "pend"
                key_result = True
            else:
                # The key is being automatically accepted, don't do anything
                # here and let the auto accept logic below handle it.
                key_result = None

            if key_result is not None:
                if self.opts.get("auth_events") is True:
                    eload = {
                        "result": key_result,
                        "act": key_act,
                        "id": load["id"],
                        "pub": load["pub"],
                    }
                    autosign_grains = load.get("autosign_grains", None)
                    if (
                        key_act in self.opts.get("auth_events_autosign_grains", [])
                        and autosign_grains
                    ):
                        eload["autosign_grains"] = autosign_grains
                    self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
                if sign_messages:
                    return self._clear_signed(
                        {"ret": key_result, "nonce": load["nonce"]},
                        sig_algo,
                    )
                else:
                    return {"enc": "clear", "load": {"ret": key_result}}

        elif key and key["state"] == "pending":
            # This key is in the pending dir and is awaiting acceptance
            if auto_reject:
                # We don't care if the keys match, this minion is being
                # auto-rejected. Move the key file from the pending dir to the
                # rejected dir.
                key["state"] = "rejected"
                self.cache.store("keys", load["id"], key)
                log.info(
                    "Pending public key for %s rejected via autoreject_file",
                    load["id"],
                )
                if self.opts.get("auth_events") is True:
                    eload = {
                        "result": False,
                        "act": "reject",
                        "id": load["id"],
                        "pub": load["pub"],
                    }
                    autosign_grains = load.get("autosign_grains", None)
                    if (
                        "reject" in self.opts.get("auth_events_autosign_grains", [])
                        and autosign_grains
                    ):
                        eload["autosign_grains"] = autosign_grains
                    self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
                if sign_messages:
                    return self._clear_signed(
                        {"ret": False, "nonce": load["nonce"]}, sig_algo
                    )
                else:
                    return {"enc": "clear", "load": {"ret": False}}

            elif not auto_sign:
                # This key is in the pending dir and is not being auto-signed.
                # Check if the keys are the same and error out if this is the
                # case. Otherwise log the fact that the minion is still
                # pending.
                if not self.compare_keys(key["pub"], load["pub"]):
                    log.error(
                        "Authentication attempt from %s failed, the public "
                        "key in pending did not match. This may be an "
                        "attempt to compromise the Salt cluster.",
                        load["id"],
                    )
                    # put denied minion key into minions_denied
                    if load["pub"] not in denied:
                        denied.append(load["pub"])
                        self.cache.store("denied_keys", load["id"], denied)
                    if self.opts.get("auth_events") is True:
                        eload = {
                            "result": False,
                            "id": load["id"],
                            "act": "denied",
                            "pub": load["pub"],
                        }
                        autosign_grains = load.get("autosign_grains", None)
                        if (
                            "denied" in self.opts.get("auth_events_autosign_grains", [])
                            and autosign_grains
                        ):
                            eload["autosign_grains"] = autosign_grains
                        self.event.fire_event(
                            eload, salt.utils.event.tagify(prefix="auth")
                        )
                    if sign_messages:
                        return self._clear_signed(
                            {"ret": False, "nonce": load["nonce"]}, sig_algo
                        )
                    else:
                        return {"enc": "clear", "load": {"ret": False}}
                else:
                    log.info(
                        "Authentication failed from host %s, the key is in "
                        "pending and needs to be accepted with salt-key "
                        "-a %s",
                        load["id"],
                        load["id"],
                    )
                    if self.opts.get("auth_events") is True:
                        eload = {
                            "result": True,
                            "act": "pend",
                            "id": load["id"],
                            "pub": load["pub"],
                        }
                        autosign_grains = load.get("autosign_grains", None)
                        if (
                            "pend" in self.opts.get("auth_events_autosign_grains", [])
                            and autosign_grains
                        ):
                            eload["autosign_grains"] = autosign_grains
                        self.event.fire_event(
                            eload, salt.utils.event.tagify(prefix="auth")
                        )
                    if sign_messages:
                        return self._clear_signed(
                            {"ret": True, "nonce": load["nonce"]}, sig_algo
                        )
                    else:
                        return {"enc": "clear", "load": {"ret": True}}
            else:
                # This key is in pending and has been configured to be
                # auto-signed. Check to see if it is the same key, and if
                # so, pass on doing anything here, and let it get automatically
                # accepted below.
                if not self.compare_keys(key["pub"], load["pub"]):
                    log.error(
                        "Authentication attempt from %s failed, the public "
                        "keys in pending did not match. This may be an "
                        "attempt to compromise the Salt cluster.",
                        load["id"],
                    )
                    # put denied minion key into minions_denied
                    if load["pub"] not in denied:
                        denied.append(load["pub"])
                        self.cache.store("denied_keys", load["id"], denied)
                    if self.opts.get("auth_events") is True:
                        eload = {
                            "result": False,
                            "act": "denied",
                            "id": load["id"],
                            "pub": load["pub"],
                        }
                        autosign_grains = load.get("autosign_grains", None)
                        if (
                            "denied" in self.opts.get("auth_events_autosign_grains", [])
                            and autosign_grains
                        ):
                            eload["autosign_grains"] = autosign_grains
                        self.event.fire_event(
                            eload, salt.utils.event.tagify(prefix="auth")
                        )
                    if sign_messages:
                        return self._clear_signed(
                            {"ret": False, "nonce": load["nonce"]}, sig_algo
                        )
                    else:
                        return {"enc": "clear", "load": {"ret": False}}
        else:
            # Something happened that I have not accounted for, FAIL!
            log.warning("Unaccounted for authentication failure")
            if self.opts.get("auth_events") is True:
                eload = {
                    "result": False,
                    "act": "error",
                    "id": load["id"],
                    "pub": load["pub"],
                }
                autosign_grains = load.get("autosign_grains", None)
                if (
                    "error" in self.opts.get("auth_events_autosign_grains", [])
                    and autosign_grains
                ):
                    eload["autosign_grains"] = autosign_grains
                self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
            if sign_messages:
                return self._clear_signed(
                    {"ret": False, "nonce": load["nonce"]}, sig_algo
                )
            else:
                return {"enc": "clear", "load": {"ret": False}}

        log.info("Authentication accepted from %s", load["id"])

        # only write to disk if you are adding the file, and in open mode,
        # which implies we accept any key from a minion.
        if (not key or key["state"] != "accepted") and not self.opts["open_mode"]:
            key = {"pub": load["pub"], "state": "accepted"}
            self.cache.store("keys", load["id"], key)
        elif self.opts["open_mode"]:
            if load["pub"] and (not key or load["pub"] != key["pub"]):
                key = {"pub": load["pub"], "state": "accepted"}
                self.cache.store("keys", load["id"], key)
            elif not load["pub"]:
                log.error("Public key is empty: %s", load["id"])
                if sign_messages:
                    return self._clear_signed(
                        {"ret": False, "nonce": load["nonce"]}, sig_algo
                    )
                else:
                    return {"enc": "clear", "load": {"ret": False}}

        pub = None

        # the con_cache is enabled, send the minion id to the cache
        if self.cache_cli:
            self.cache_cli.put_cache([load["id"]])

        # The key payload may sometimes be corrupt when using auto-accept
        # and an empty request comes in
        try:
            pub = salt.crypt.PublicKey.from_str(key["pub"])
        except Exception as err:  # pylint: disable=broad-except
            log.error(
                'Corrupt or missing public key "%s": %s',
                load["id"],
                err,
                exc_info_on_loglevel=logging.DEBUG,
            )
            if sign_messages:
                return self._clear_signed(
                    {"ret": False, "nonce": load["nonce"]}, sig_algo
                )
            else:
                return {"enc": "clear", "load": {"ret": False}}

        ret = {
            "enc": "pub",
            "pub_key": self.master_key.get_pub_str(),
            "publish_port": self.opts["publish_port"],
        }

        # sign the master's pubkey (if enabled) before it is
        # sent to the minion that was just authenticated
        if self.opts["master_sign_pubkey"]:
            # append the pre-computed signature to the auth-reply
            if self.master_key.pubkey_signature:
                log.debug("Adding pubkey signature to auth-reply")
                log.debug(self.master_key.pubkey_signature)
                ret.update({"pub_sig": self.master_key.pubkey_signature})
            else:
                # the master has its own signing-keypair, compute the master.pub's
                # signature and append that to the auth-reply
                log.debug("Signing master public key before sending")
                pub_sign = self.master_key.sign_key.sign(
                    ret["pub_key"], algorithm=sig_algo
                )
                ret.update({"pub_sig": binascii.b2a_base64(pub_sign)})

        if self.opts["auth_mode"] >= 2:
            if "token" in load:
                try:
                    mtoken = self.master_key.decrypt(load["token"], enc_algo)
                    aes = "{}_|-{}".format(
                        salt.master.SMaster.secrets["aes"]["secret"].value, mtoken
                    )
                except UnsupportedAlgorithm as exc:
                    log.info(
                        "Minion %s tried to authenticate with unsupported encryption algorithm: %s",
                        load["id"],
                        enc_algo,
                    )
                    return {"enc": "clear", "load": {"ret": "bad enc algo"}}
                except Exception as exc:  # pylint: disable=broad-except
                    log.warning("Token failed to decrypt %s", exc)
                    # Token failed to decrypt, send back the salty bacon to
                    # support older minions
            else:
                aes = self.aes_key

            ret["aes"] = pub.encrypt(aes, enc_algo)
            ret["session"] = pub.encrypt(self.session_key(load["id"]), enc_algo)
        else:
            if "token" in load:
                try:
                    mtoken = self.master_key.decrypt(load["token"], enc_algo)
                    ret["token"] = pub.encrypt(mtoken, enc_algo)
                except UnsupportedAlgorithm as exc:
                    log.info(
                        "Minion %s tried to authenticate with unsupported encryption algorithm: %s",
                        load["id"],
                        enc_algo,
                    )
                    return {"enc": "clear", "load": {"ret": "bad enc algo"}}
                except Exception as exc:  # pylint: disable=broad-except
                    # Token failed to decrypt, send back the salty bacon to
                    # support older minions
                    log.warning("Token failed to decrypt: %r", exc)

            aes = self.aes_key
            ret["aes"] = pub.encrypt(aes, enc_algo)
            ret["session"] = pub.encrypt(self.session_key(load["id"]), enc_algo)

        if version < 3:
            log.warning(
                "Minion using legacy request server protocol, please upgrade %s",
                load["id"],
            )

        # Be aggressive about the signature
        digest = salt.utils.stringutils.to_bytes(hashlib.sha256(aes).hexdigest())
        ret["sig"] = self.master_key.encrypt(digest)
        if self.opts.get("auth_events") is True:
            eload = {
                "result": True,
                "act": "accept",
                "id": load["id"],
                "pub": load["pub"],
            }
            autosign_grains = load.get("autosign_grains", None)
            if (
                "accept" in self.opts.get("auth_events_autosign_grains", [])
                and autosign_grains
            ):
                eload["autosign_grains"] = autosign_grains
            self.event.fire_event(eload, salt.utils.event.tagify(prefix="auth"))
        if sign_messages:
            ret["nonce"] = load["nonce"]
            return self._clear_signed(ret, sig_algo)
        return ret

    def close(self):
        self.transport.close()
        if self.event is not None:
            self.event.destroy()


class PoolRoutingChannel:
    """
    Request channel that routes incoming messages to per-pool worker processes
    using transport-native IPC (the pooled path).

    This class is returned by :meth:`ReqServerChannel.factory` when
    ``worker_pools_enabled=True`` (the default).  It is mutually exclusive
    with the plain :class:`ReqServerChannel` — only one of the two is ever
    active for a given master process.

    Architecture::

        External Transport → PoolRoutingChannel → RequestClient (IPC) →
        Pool RequestServer (IPC) → MWorkers

    ``_auth`` handling
    ------------------
    Under a fully started master, ``_auth`` is looked up in the routing table
    and forwarded to the mapped pool's IPC RequestServer, then handled in a
    worker by :meth:`~salt.master.MWorker._handle_clear` →
    :meth:`~salt.master.ClearFuncs._auth`.

    If the pool's IPC client is not connected yet (e.g. tests calling
    :meth:`handle_message` without ``post_fork``), clear-text ``_auth`` is
    handled inline with the same logic as :meth:`ReqServerChannel.handle_message`.

    See :meth:`ReqServerChannel.factory` for the authoritative description of
    the two mutually exclusive paths.

    Key advantages over the legacy single-pool design:
    - No multiprocessing.Queue overhead
    - Uses transport-native IPC (ZeroMQ/TCP/WebSocket)
    - Clean separation of concerns
    - Works across all transports without transport modifications
    """

    def __init__(self, opts, transport, worker_pools):
        """
        Initialize the pool routing channel.

        Args:
            opts: Master configuration options
            transport: The external transport instance (port 4506)
            worker_pools: Dict of pool configurations {pool_name: config}
        """
        self.opts = opts
        self.transport = transport
        self.worker_pools = worker_pools
        self.pool_clients = {}  # pool_name -> RequestClient
        self.pool_servers = {}  # pool_name -> RequestServer
        self.io_loop = None
        self.event = None
        self.router = None
        self.crypticle = None
        self.master_key = None
        self.auto_key = None

        (pathlib.Path(self.opts["cachedir"]) / "sessions").mkdir(exist_ok=True)
        self.sessions = {}

        # Same key cache / minion bookkeeping as ReqServerChannel so clear-text
        # _auth can run inline when IPC pool clients are not yet connected
        # (functional tests and bootstrap scenarios).
        self.cache = salt.cache.Cache(opts, driver=self.opts["keys.cache_driver"])
        if self.opts["con_cache"]:
            self.cache_cli = CacheCli(self.opts)
        else:
            self.cache_cli = False
            self.ckminions = salt.utils.minions.CkMinions(self.opts)

        # Build routing table for command-based routing
        self._build_routing_table()

        log.info(
            "PoolRoutingChannel initialized with pools: %s",
            list(worker_pools.keys()),
        )

    def _build_routing_table(self):
        """
        Build command-to-pool routing table from configuration.

        Exactly one pool must include ``"*"`` in its commands and becomes
        :attr:`default_pool`.  Pool configuration is validated during master
        startup (see
        :func:`salt.config.worker_pools.validate_worker_pools_config`), so
        this method only translates the validated layout into the lookup
        table used at routing time.
        """
        self.command_to_pool = {}
        self.default_pool = None

        for pool_name, config in self.worker_pools.items():
            for cmd in config.get("commands", []):
                if cmd == "*":
                    self.default_pool = pool_name
                else:
                    self.command_to_pool[cmd] = pool_name

        if self.worker_pools and not self.default_pool:
            raise ValueError(
                "Worker pool configuration must have exactly one pool with "
                "catchall ('*') in its commands."
            )

    @property
    def aes_key(self):
        if self.opts.get("cluster_id", None):
            return salt.master.SMaster.secrets["cluster_aes"]["secret"].value
        return salt.master.SMaster.secrets["aes"]["secret"].value

    def session_key(self, minion):
        """
        Returns a session key for the given minion id.
        """
        now = time.time()
        if minion in self.sessions:
            if now - self.sessions[minion][0] < self.opts["publish_session"]:
                return self.sessions[minion][1]

        path = pathlib.Path(self.opts["cachedir"]) / "sessions" / minion
        try:
            if now - path.stat().st_mtime > self.opts["publish_session"]:
                salt.crypt.Crypticle.write_key(path)
        except FileNotFoundError:
            salt.crypt.Crypticle.write_key(path)

        self.sessions[minion] = (
            path.stat().st_mtime,
            salt.crypt.Crypticle.read_key(path),
        )
        return self.sessions[minion][1]

    def _update_aes(self):
        """
        Check to see if a fresh AES key is available and update the components
        of the worker
        """
        key = "aes"
        if self.opts.get("cluster_id", None):
            key = "cluster_aes"

        if (
            salt.master.SMaster.secrets[key]["secret"].value
            != self.crypticle.key_string
        ):
            self.crypticle = _get_crypticle(
                self.opts, salt.master.SMaster.secrets[key]["secret"].value
            )
            return True
        return False

    def pre_fork(self, process_manager, *args, **kwargs):
        """
        Pre-fork setup: Initialize external transport and create RequestServer
        for each worker pool on IPC.
        """
        import salt.master
        import salt.transport.base
        from salt.utils.channel import create_server_transport

        # Pass secrets if not present (critical for decryption in routing)
        if "secrets" not in kwargs:
            kwargs["secrets"] = salt.master.SMaster.secrets

        # Setup external transport (this binds the actual network ports 4505/4506)
        if hasattr(self.transport, "pre_fork"):
            self.transport.pre_fork(process_manager, *args, **kwargs)

        # Create a RequestServer for each pool on IPC
        for pool_name, config in self.worker_pools.items():
            # Create pool-specific opts for IPC
            pool_opts = self.opts.copy()
            pool_opts["pool_name"] = pool_name
            # Disable worker pools for internal routing to avoid circular dependency
            pool_opts["worker_pools_enabled"] = False

            # Configure IPC for this pool
            if pool_opts.get("ipc_mode") == "tcp":
                # TCP IPC mode: use unique port per pool
                base_port = pool_opts.get("tcp_master_workers", 4515)
                port_offset = zlib.adler32(pool_name.encode()) % 1000
                pool_opts["ret_port"] = base_port + port_offset
                log.info(
                    "Pool '%s' RequestServer using TCP IPC on port %d",
                    pool_name,
                    pool_opts["ret_port"],
                )
            else:
                # Standard IPC mode: use unique socket per pool
                sock_dir = pool_opts.get("sock_dir", "/tmp/salt")
                os.makedirs(sock_dir, exist_ok=True)
                pool_opts["workers_ipc_name"] = f"workers-{pool_name}.ipc"
                log.debug(
                    "Pool '%s' RequestServer using IPC socket: %s",
                    pool_name,
                    pool_opts["workers_ipc_name"],
                )

            # Create RequestServer for this pool using transport factory
            try:
                pool_transport = create_server_transport(pool_opts)
                # We wrap it in a minimal ReqServerChannel for compatibility
                pool_server = ReqServerChannel(pool_opts, pool_transport)
                pool_server.pre_fork(process_manager, *args, **kwargs)
                self.pool_servers[pool_name] = pool_server
                log.info("Created RequestServer for pool '%s'", pool_name)
            except Exception as exc:  # pylint: disable=broad-except
                log.error(
                    "Failed to create RequestServer for pool '%s': %s", pool_name, exc
                )
                raise

        log.info(
            "PoolRoutingChannel pre_fork complete for %d pools", len(self.worker_pools)
        )

    def post_fork(self, payload_handler, io_loop, **kwargs):
        """
        Post-fork setup in the routing process.

        This is where we:
        1. Set up the master infrastructure (crypticle, events, keys)
        2. Create RequestClient connections to each pool's RequestServer
        3. Connect the external transport to our routing handler
        """
        pool_name = kwargs.get("pool_name")
        if pool_name:
            # We are in an MWorker process for a specific pool.
            # Delegate to the pool's RequestServer.
            if pool_name in self.pool_servers:
                pool_server = self.pool_servers[pool_name]
                return pool_server.post_fork(payload_handler, io_loop, **kwargs)
            else:
                log.error("Pool '%s' not found in pool_servers", pool_name)
                return

        import salt.master
        from salt.utils.channel import create_request_client

        self.io_loop = io_loop

        # Setup master infrastructure (same as ReqServerChannel)
        if (
            self.opts.get("pub_server_niceness")
            and not salt.utils.platform.is_windows()
        ):
            log.debug(
                "setting Publish daemon niceness to %i",
                self.opts["pub_server_niceness"],
            )
            os.nice(self.opts["pub_server_niceness"])

        # Create event manager for the routing process
        self.event = salt.utils.event.get_master_event(
            self.opts, self.opts["sock_dir"], listen=False, io_loop=io_loop
        )

        # Set up crypticle for payload decryption during routing
        self.crypticle = _get_crypticle(self.opts, self.aes_key)

        self.master_key = salt.crypt.MasterKeys(self.opts)

        # Create RequestClient for each pool (connects to pool's IPC RequestServer)
        for pool_name in self.worker_pools.keys():
            # Create pool-specific opts matching the pool's RequestServer
            pool_opts = self.opts.copy()
            pool_opts["pool_name"] = pool_name
            # Disable worker pools for internal routing to avoid circular dependency
            pool_opts["worker_pools_enabled"] = False

            if pool_opts.get("ipc_mode") == "tcp":
                # TCP IPC: connect to pool's port
                base_port = pool_opts.get("tcp_master_workers", 4515)
                port_offset = zlib.adler32(pool_name.encode()) % 1000
                pool_opts["ret_port"] = base_port + port_offset
                pool_opts["master_uri"] = f"tcp://127.0.0.1:{pool_opts['ret_port']}"
                log.debug(
                    "Pool '%s' client connecting to TCP port %d",
                    pool_name,
                    pool_opts["ret_port"],
                )
            else:
                # IPC socket: connect to pool's socket
                pool_opts["workers_ipc_name"] = f"workers-{pool_name}.ipc"
                ipc_path = os.path.join(
                    self.opts["sock_dir"], pool_opts["workers_ipc_name"]
                )
                pool_opts["master_uri"] = f"ipc://{ipc_path}"
                log.debug(
                    "Pool '%s' client connecting to IPC socket: %s",
                    pool_name,
                    pool_opts["workers_ipc_name"],
                )

            try:
                # Use our dedicated request client factory for routing
                client = create_request_client(pool_opts, io_loop)
                self.pool_clients[pool_name] = client
                log.info("Created RequestClient for pool '%s'", pool_name)
            except Exception as exc:  # pylint: disable=broad-except
                log.error(
                    "Failed to create RequestClient for pool '%s': %s", pool_name, exc
                )
                raise

        # Connect external transport to our routing handler
        if hasattr(self.transport, "post_fork"):
            self.transport.post_fork(self.handle_and_route_message, io_loop, **kwargs)

        log.info(
            "PoolRoutingChannel post_fork complete with %d pool clients",
            len(self.pool_clients),
        )

    def _req_channel_auth_delegate(self):
        """
        Build a minimal :class:`ReqServerChannel` view for running
        :meth:`ReqServerChannel._auth` with this channel's opts, keys, and
        cache (used when pool IPC clients are not connected yet).
        """
        ch = ReqServerChannel.__new__(ReqServerChannel)
        ch.opts = self.opts
        ch.transport = self.transport
        ch.cache = self.cache
        ch.event = self.event
        ch.master_key = self.master_key
        ch.sessions = self.sessions
        ch.auto_key = getattr(self, "auto_key", None)
        ch.cache_cli = getattr(self, "cache_cli", False)
        ch.ckminions = getattr(self, "ckminions", None)
        ch.crypticle = getattr(self, "crypticle", None)
        return ch

    async def _handle_clear_auth_local(self, payload, version):
        """
        Run clear-text ``_auth`` the same way :meth:`ReqServerChannel.handle_message`
        does, without forwarding to a worker pool (no IPC client yet).
        """
        proxy = self._req_channel_auth_delegate()
        try:
            payload = ReqServerChannel._decode_payload(proxy, payload, version)
        except Exception as exc:  # pylint: disable=broad-except
            exc_type = type(exc).__name__
            if exc_type == "AuthenticationError":
                log.debug(
                    "Minion failed to auth to master. Since the payload is "
                    "encrypted, it is not known which minion failed to "
                    "authenticate. It is likely that this is a transient "
                    "failure due to the master rotating its public key."
                )
            else:
                log.error("Bad load from minion: %s: %s", exc_type, exc)
            return "bad load"

        if not isinstance(payload, dict) or not isinstance(payload.get("load"), dict):
            log.error(
                "payload and load must be a dict. Payload was: %s",
                payload,
            )
            return "payload and load must be a dict"

        try:
            id_ = payload["load"].get("id", "")
            if "\0" in id_:
                log.error("Payload contains an id with a null byte: %s", payload)
                return "bad load: id contains a null byte"
        except TypeError:
            log.error("Payload contains non-string id: %s", payload)
            return f"bad load: id {id_} is not a string"

        sign_messages = version > 1

        if (
            payload.get("enc") == "clear"
            and payload.get("load", {}).get("cmd") == "_auth"
        ):
            start = time.time()
            ret = ReqServerChannel._auth(proxy, payload["load"], sign_messages, version)
            if self.opts.get("master_stats", False) and getattr(
                self, "payload_handler", None
            ):
                await self.payload_handler({"cmd": "_auth", "_start": start})
            return ret

        log.error("clear-auth local handler called for non-auth payload: %s", payload)
        return {"error": "Internal routing error", "success": False}

    async def handle_and_route_message(self, payload):
        """
        Route an incoming request to the appropriate worker pool (pooled path).

        Determines the target pool by inspecting the ``cmd`` field of the
        payload load (decrypting first if the load is encrypted), looks it up
        in the routing table, then forwards the raw payload to that pool's
        IPC RequestServer via a RequestClient.

        Clear-text ``_auth`` is normally routed like any other command. When
        no IPC client exists for the target pool yet (e.g. functional tests
        that call :meth:`handle_message` without a full ``post_fork``), it is
        handled inline using the same logic as :meth:`ReqServerChannel.handle_message`.

        See :class:`PoolRoutingChannel` and :meth:`ReqServerChannel.factory`
        for the full explanation of the two mutually exclusive request paths.
        """
        if (
            not isinstance(payload, dict)
            or "enc" not in payload
            or "load" not in payload
        ):
            log.warning("bad load received on socket")
            return "bad load"
        try:
            version = int(payload.get("version", 0))
        except ValueError:
            version = 0

        # Enforce minimum authentication protocol version to prevent downgrade attacks
        minimum_version = self.opts.get("minimum_auth_version", 0)
        if minimum_version > 0 and version < minimum_version:
            load = payload.get("load")
            if isinstance(load, dict):
                minion_id = load.get("id", "unknown minion")
            else:
                minion_id = "unknown minion"
            log.warning(
                "Rejected authentication attempt from minion '%s' using "
                "protocol version %d (minimum required: %d)",
                minion_id,
                version,
                minimum_version,
            )
            return "bad load"

        try:
            # Simple command-based routing from our routing table
            load = payload.get("load", {})
            if isinstance(load, dict):
                cmd = load.get("cmd", "unknown")
            else:
                # This is likely an encrypted payload. We need to decrypt
                # to determine the command for routing.
                try:
                    # Determine which key to use based on the 'enc' field
                    enc = payload.get("enc", "aes")
                    if enc == "aes":
                        import salt.master

                        key = (
                            salt.master.SMaster.secrets.get("aes", {})
                            .get("secret", {})
                            .value
                        )
                        if key:
                            import salt.crypt

                            crypticle = salt.crypt.Crypticle(self.opts, key)
                            decrypted = crypticle.loads(load)
                            if isinstance(decrypted, dict) and "cmd" in decrypted:
                                cmd = decrypted.get("cmd", "unknown")
                            elif isinstance(decrypted, dict) and "load" in decrypted:
                                cmd = decrypted["load"].get("cmd", "unknown")
                            else:
                                cmd = "unknown"
                        else:
                            cmd = "unknown"
                    elif enc == "pub":
                        # RSA encryption
                        import salt.crypt

                        mkey = salt.crypt.MasterKeys(self.opts)
                        decrypted = mkey.priv_decrypt(load)
                        if isinstance(decrypted, bytes):
                            import salt.payload

                            decrypted = salt.payload.loads(decrypted)
                        if isinstance(decrypted, dict) and "cmd" in decrypted:
                            cmd = decrypted.get("cmd", "unknown")
                        elif isinstance(decrypted, dict) and "load" in decrypted:
                            cmd = decrypted["load"].get("cmd", "unknown")
                        else:
                            cmd = "unknown"
                    else:
                        cmd = "unknown"
                except Exception:  # pylint: disable=broad-except
                    cmd = "unknown"

            pool_name = self.command_to_pool.get(cmd, self.default_pool)

            log.debug(
                "Routing: cmd=%s -> pool='%s' (pools: %s)",
                cmd,
                pool_name,
                list(self.worker_pools.keys()),
            )

            if pool_name not in self.pool_clients:
                if (
                    payload.get("enc") == "clear"
                    and isinstance(payload.get("load"), dict)
                    and payload["load"].get("cmd") == "_auth"
                ):
                    return await self._handle_clear_auth_local(payload, version)
                log.error(
                    "No client available for pool '%s'. Available: %s",
                    pool_name,
                    list(self.pool_clients.keys()),
                )
                return {"error": f"No client for pool {pool_name}"}

            # Forward to the appropriate pool's RequestServer via IPC
            client = self.pool_clients[pool_name]
            reply = await client.send(payload)

            return reply

        except Exception as exc:  # pylint: disable=broad-except
            log.error(
                "Error in pool routing: %s",
                exc,
                exc_info=True,
            )
            return {"error": "Internal routing error", "success": False}

    # Alias for compatibility with older tests and code that expect handle_message
    handle_message = handle_and_route_message

    def close(self):
        """
        Close all resources: pool clients, pool servers, event manager, and external transport.
        """
        log.info("Closing PoolRoutingChannel")

        # Close all pool clients (RequestClients to pool RequestServers)
        for pool_name, client in self.pool_clients.items():
            try:
                if hasattr(client, "close"):
                    client.close()
                elif hasattr(client, "destroy"):
                    client.destroy()
            except Exception as exc:  # pylint: disable=broad-except
                log.error("Error closing client for pool '%s': %s", pool_name, exc)
        self.pool_clients.clear()

        # Close all pool servers
        for pool_name, server in self.pool_servers.items():
            try:
                if hasattr(server, "close"):
                    server.close()
            except Exception as exc:  # pylint: disable=broad-except
                log.error("Error closing server for pool '%s': %s", pool_name, exc)
        self.pool_servers.clear()

        # Close event manager
        if self.event is not None:
            try:
                self.event.destroy()
            except Exception as exc:  # pylint: disable=broad-except
                log.error("Error closing event manager: %s", exc)
            self.event = None

        # Close external transport
        if hasattr(self.transport, "close"):
            try:
                self.transport.close()
            except Exception as exc:  # pylint: disable=broad-except
                log.error("Error closing external transport: %s", exc)

        log.info("PoolRoutingChannel closed")


class PubServerChannel:
    """
    Factory class to create subscription channels to the master's Publisher
    """

    @classmethod
    def factory(cls, opts, **kwargs):
        if "master_uri" not in opts and "master_uri" in kwargs:
            opts["master_uri"] = kwargs["master_uri"]
        presence_events = False
        if opts.get("presence_events", False):
            tcp_only = True
            for transport, _ in salt.utils.channel.iter_transport_opts(opts):
                if transport != "tcp":
                    tcp_only = False
            if tcp_only:
                # Only when the transport is TCP only, the presence events will
                # be handled here. Otherwise, it will be handled in the
                # 'Maintenance' process.
                presence_events = True
        transport = salt.transport.publish_server(opts, **kwargs)
        return cls(opts, transport, presence_events=presence_events)

    def __init__(self, opts, transport, presence_events=False):
        self.opts = opts
        self.ckminions = salt.utils.minions.CkMinions(self.opts)
        self.transport = transport
        self.aes_funcs = salt.master.AESFuncs(self.opts)
        self.present = {}
        self.presence_events = presence_events
        self.event = salt.utils.event.get_event("master", opts=self.opts, listen=False)

    @property
    def aes_key(self):
        if self.opts.get("cluster_id", None):
            return salt.master.SMaster.secrets["cluster_aes"]["secret"].value
        return salt.master.SMaster.secrets["aes"]["secret"].value

    def __getstate__(self):
        return {
            "opts": self.opts,
            "transport": self.transport,
            "presence_events": self.presence_events,
        }

    def __setstate__(self, state):
        self.opts = state["opts"]
        self.state = state["presence_events"]
        self.transport = state["transport"]
        self.event = salt.utils.event.get_event("master", opts=self.opts, listen=False)
        self.ckminions = salt.utils.minions.CkMinions(self.opts)
        self.present = {}
        self.master_key = salt.crypt.MasterKeys(self.opts)

    def close(self):
        self.transport.close()
        if self.event is not None:
            self.event.destroy()
            self.event = None
        if self.aes_funcs is not None:
            self.aes_funcs.destroy()
            self.aes_funcs = None

    def pre_fork(self, process_manager, *args, **kwargs):
        """
        Do anything necessary pre-fork. Since this is on the master side this will
        primarily be used to create IPC channels and create our daemon process to
        do the actual publishing

        :param func process_manager: A ProcessManager, from salt.utils.process.ProcessManager
        """
        if hasattr(self.transport, "publish_daemon"):
            # Extract kwargs for the process.
            # We check for a named 'kwargs' key first (from salt/master.py),
            # then fallback to the entire kwargs dict.
            proc_kwargs = kwargs.pop("kwargs", kwargs).copy()
            if "secrets" not in proc_kwargs:
                import salt.master

                proc_kwargs["secrets"] = salt.master.SMaster.secrets
            if "started" not in proc_kwargs:
                proc_kwargs["started"] = self.transport.started
            process_manager.add_process(self._publish_daemon, kwargs=proc_kwargs)

    def _publish_daemon(self, **kwargs):
        import salt.master

        if self.opts["pub_server_niceness"] and not salt.utils.platform.is_windows():
            log.debug(
                "setting Publish daemon niceness to %i",
                self.opts["pub_server_niceness"],
            )
            os.nice(self.opts["pub_server_niceness"])
        secrets = kwargs.pop("secrets", None)
        started = kwargs.pop("started", None)
        if secrets is not None:
            salt.master.SMaster.secrets = secrets
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.transport.publish_daemon(
            self.publish_payload,
            self.presence_callback,
            self.remove_presence_callback,
            secrets=secrets,
            started=started,
        )

    def presence_callback(self, subscriber, msg):
        if msg["enc"] != "aes":
            # We only accept 'aes' encoded messages for 'id'
            return
        crypticle = _get_crypticle(self.opts, self.aes_key)
        load = crypticle.loads(msg["load"])
        load = salt.transport.frame.decode_embedded_strs(load)
        if not self.aes_funcs.verify_minion(load["id"], load["tok"]):
            return
        subscriber.id_ = load["id"]
        self._add_client_present(subscriber)

    def remove_presence_callback(self, subscriber):
        self._remove_client_present(subscriber)

    def _add_client_present(self, client):
        id_ = client.id_
        if id_ in self.present:
            clients = self.present[id_]
            clients.add(client)
        else:
            self.present[id_] = {client}
            if self.presence_events:
                data = {"new": [id_], "lost": []}
                self.event.fire_event(
                    data, salt.utils.event.tagify("change", "presence")
                )
                data = {"present": list(self.present.keys())}
                self.event.fire_event(
                    data, salt.utils.event.tagify("present", "presence")
                )

    def _remove_client_present(self, client):
        id_ = client.id_
        if id_ is None or id_ not in self.present:
            # This is possible if _remove_client_present() is invoked
            # before the minion's id is validated.
            return

        clients = self.present[id_]
        if client not in clients:
            # Since _remove_client_present() is potentially called from
            # _stream_read() and/or publish_payload(), it is possible for
            # it to be called twice, in which case we will get here.
            # This is not an abnormal case, so no logging is required.
            return

        clients.remove(client)
        if len(clients) == 0:
            del self.present[id_]
            if self.presence_events:
                data = {"new": [], "lost": [id_]}
                self.event.fire_event(
                    data, salt.utils.event.tagify("change", "presence")
                )
                data = {"present": list(self.present.keys())}
                self.event.fire_event(
                    data, salt.utils.event.tagify("present", "presence")
                )

    async def publish_payload(self, load, *args):
        load = salt.payload.loads(load)
        unpacked_package = self.wrap_payload(load)
        try:
            payload = salt.payload.loads(unpacked_package["payload"])
        except KeyError:
            log.error("Invalid package %r", unpacked_package)
            raise
        payload = salt.payload.dumps(payload)
        if "topic_lst" in unpacked_package:
            topic_list = unpacked_package["topic_lst"]
            ret = await self.transport.publish_payload(payload, topic_list)
        else:
            ret = await self.transport.publish_payload(payload)
        return ret

    def wrap_payload(self, load):
        payload = {"enc": "aes"}
        if not self.opts.get("cluster_id", None):
            load["serial"] = salt.master.SMaster.get_serial()
        crypticle = _get_crypticle(self.opts, self.aes_key)
        payload["load"] = crypticle.dumps(load)
        if self.opts["sign_pub_messages"]:
            log.debug("Signing data packet")
            payload["sig_algo"] = self.opts["publish_signing_algorithm"]
            payload["sig"] = self.master_key.sign(
                payload["load"], self.opts["publish_signing_algorithm"]
            )

        int_payload = {"payload": salt.payload.dumps(payload)}

        # If topics are upported, target matching has to happen master side
        match_targets = ["pcre", "glob", "list"]
        if self.transport.topic_support() and load["tgt_type"] in match_targets:
            # add some targeting stuff for lists only (for now)
            if load["tgt_type"] == "list":
                int_payload["topic_lst"] = load["tgt"]
            if isinstance(load["tgt"], str):
                # Fetch a list of minions that match
                _res = self.ckminions.check_minions(
                    load["tgt"], tgt_type=load["tgt_type"]
                )
                match_ids = _res["minions"]
                log.debug("Publish Side Match: %s", match_ids)
                # Send list of miions thru so zmq can target them
                int_payload["topic_lst"] = match_ids
            else:
                int_payload["topic_lst"] = load["tgt"]

        return int_payload

    async def publish(self, load):
        """
        Publish "load" to minions
        """
        log.debug(
            "Sending payload to publish daemon. jid=%s load=%s",
            load.get("jid", None),
            repr(load)[:40],
        )
        payload = salt.payload.dumps(load)
        await self.transport.publish(payload)


class MasterPubServerChannel:
    """ """

    @classmethod
    def factory(cls, opts, **kwargs):
        _discover_event = kwargs.get("_discover_event", None)

        if opts.get("cluster_id"):
            # Cluster mode: Use TCP-based transport for peer communication while
            # preserving normal local IPC behavior for internal processes.
            port = opts.get("cluster_port", 55596)
            pull_path = os.path.join(opts["sock_dir"], "master_event_pull.ipc")
            pub_path = os.path.join(opts["sock_dir"], "master_event_pub.ipc")
            bind_host = opts.get("interface", "127.0.0.1")

            try:
                transport = salt.transport.tcp.PublishServer(
                    opts,
                    pub_host=bind_host,
                    pub_port=opts.get("publish_port", 4505),
                    pub_path=pub_path,
                    pull_host=bind_host,
                    pull_port=port,
                    pull_path=pull_path,
                )
            except OSError as exc:
                if exc.errno == errno.EADDRINUSE:
                    transport = salt.transport.tcp.PublishServer(
                        opts,
                        pub_host=bind_host,
                        pub_port=opts.get("publish_port", 4505),
                        pub_path=pub_path,
                        pull_host=bind_host,
                        pull_port=0,
                        pull_path=pull_path,
                    )
                else:
                    raise
        else:
            transport = salt.transport.ipc_publish_server("master", opts)

        return cls(opts, transport, _discover_event=_discover_event)

    def __init__(
        self,
        opts,
        transport,
        presence_events=False,
        _discover_event=None,
        _discover_token=None,
    ):
        self.opts = opts
        self.transport = transport
        self.io_loop = tornado.ioloop.IOLoop.current()
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.peer_keys = {}
        self.cluster_peers = self.opts["cluster_peers"]
        self._discover_event = _discover_event
        self._discover_token = _discover_token
        self._discover_candidates = {}

    def gen_token(self):
        return "".join(random.choices(string.ascii_letters + string.digits, k=32))

    def discover_peers(self):
        path = self.master_key.master_pub_path
        with salt.utils.files.fopen(path, "r") as fp:
            pub = fp.read()

        self._discover_token = self.gen_token()

        for peer in self.cluster_peers:
            log.error("Discover cluster from %s", peer)
            tosign = salt.payload.package(
                {
                    "peer_id": self.opts["id"],
                    "pub": pub,
                    "token": self._discover_token,
                }
            )
            key = salt.crypt.PrivateKeyString(self.private_key())
            sig = key.sign(tosign)
            data = {
                "sig": sig,
                "payload": tosign,
            }
            with salt.utils.event.get_master_event(
                self.opts, self.opts["sock_dir"], listen=False
            ) as event:
                success = event.fire_event(
                    data,
                    salt.utils.event.tagify("discover", "peer", "cluster"),
                    timeout=30000,  # 30 second timeout
                )
                if not success:
                    log.error("Unable to send aes key event")

    def send_aes_key_event(self):
        log.debug("Sending AES key event")
        data = {"peer_id": self.opts["id"], "peers": {}}
        for peer in self.cluster_peers:
            peer_pub = (
                pathlib.Path(self.opts["cluster_pki_dir"]) / "peers" / f"{peer}.pub"
            )
            if peer_pub.exists():
                pub = salt.crypt.PublicKey.from_file(peer_pub)
                aes = salt.master.SMaster.secrets["aes"]["secret"].value
                digest = salt.utils.stringutils.to_bytes(
                    hashlib.sha256(aes).hexdigest()
                )
                data["peers"][peer] = {
                    "aes": pub.encrypt(aes, algorithm="OAEP-SHA224"),
                    "sig": self.master_key.master_key.encrypt(digest),
                }
            else:
                log.warning("Peer key missing %r", peer_pub)
                # request peer key
                data["peers"][peer] = {}
        with salt.utils.event.get_master_event(
            self.opts, self.opts["sock_dir"], listen=False
        ) as event:
            success = event.fire_event(
                data,
                salt.utils.event.tagify(self.opts["id"], "peer", "cluster"),
                timeout=30000,  # 30 second timeout
            )
            if not success:
                log.error("Unable to send aes key event")

    def __getstate__(self):
        return {
            "opts": self.opts,
            "transport": self.transport,
            "_discover_event": self._discover_event,
        }

    def __setstate__(self, state):
        self.opts = state["opts"]
        self.transport = state["transport"]
        self._discover_event = state["_discover_event"]

    def close(self):
        self.transport.close()

    def pre_fork(self, process_manager, *args, **kwargs):
        """
        Do anything necessary pre-fork. Since this is on the master side this will
        primarily be used to create IPC channels and create our daemon process to
        do the actual publishing

        :param func process_manager: A ProcessManager, from salt.utils.process.ProcessManager
        """
        if hasattr(self.transport, "publish_daemon"):
            proc_kwargs = kwargs.pop("kwargs", kwargs)
            process_manager.add_process(
                self._publish_daemon, kwargs=proc_kwargs, name="EventPublisher"
            )

    def _publish_daemon(self, **kwargs):
        """Clean implementation: separate local IPC from cluster peer communication."""
        if (
            self.opts.get("event_publisher_niceness")
            and not salt.utils.platform.is_windows()
        ):
            log.info(
                "setting EventPublisher niceness to %i",
                self.opts["event_publisher_niceness"],
            )
            os.nice(self.opts["event_publisher_niceness"])

        self.io_loop = tornado.ioloop.IOLoop.current()

        # Always set up the local IPC-based event publisher first
        # This ensures internal processes (like pytest_engine) can communicate reliably
        if hasattr(self.transport, "publisher"):
            aio_loop = salt.utils.asynchronous.aioloop(self.io_loop)
            aio_loop.create_task(
                self.transport.publisher(
                    self.publish_payload,
                    io_loop=self.io_loop,
                )
            )

        # Initialize cluster peer state unconditionally so that non-cluster
        # masters also have an empty ``pushers`` list -- publish_payload
        # iterates ``self.pushers`` on every event.
        self.pushers = []

        # Cluster-specific peer communication (separate from local IPC)
        if self.opts.get("cluster_id"):
            self.tcp_master_pool_port = self.opts.get("cluster_port", 55596)
            self.auth_errors = {}
            self.peer_map = {}

            for peer in self.opts.get("cluster_peers", []):
                host, port = peer.rsplit(":", 1) if ":" in peer else (peer, 55596)
                pusher = self.pusher(host, int(port))
                self.pushers.append(pusher)
                self.auth_errors[host] = collections.deque()

            # Set up the cluster pool puller for incoming peer events
            self.pool_puller = salt.transport.tcp.TCPPuller(
                host=self.opts.get("interface", "127.0.0.1"),
                port=self.tcp_master_pool_port,
                io_loop=self.io_loop,
                payload_handler=self.handle_pool_publish,
            )
            self.pool_puller.start()

            if self.opts.get("cluster_peers"):
                self.io_loop.call_later(2.0, self.discover_peers)

        # run forever
        try:
            self.io_loop.start()
        except (KeyboardInterrupt, SystemExit):
            pass
        finally:
            self.close()

    def private_key(self):
        """
        The public key string associated with this node.
        """
        # XXX Do not read every time
        path = self.master_key.master_rsa_path
        with salt.utils.files.fopen(path, "r") as fp:
            return fp.read()

    def public_key(self):
        """
        The public key string associated with this node.
        """
        # XXX Do not read every time
        path = self.master_key.master_pub_path
        with salt.utils.files.fopen(path, "r") as fp:
            return fp.read()

    def cluster_key(self):
        """
        The private key associated with this cluster.
        """
        # XXX Do not read every time
        path = pathlib.Path(self.master_key.cluster_rsa_path)
        if path.exists():
            return path.read_text(encoding="utf-8")

    def cluster_public_key(self):
        """
        The private key associated with this cluster.
        """
        # XXX Do not read every time
        path = pathlib.Path(self.master_key.cluster_pub_path)
        if path.exists():
            return path.read_text(encoding="utf-8")

    def pusher(self, peer, port=None):
        if port is None:
            port = self.tcp_master_pool_port
        return salt.transport.tcp.PublishServer(
            self.opts,
            pull_host=peer,
            pull_port=port,
        )

    async def handle_pool_publish(self, payload):
        """
        Handle incoming events from cluster peer.
        """
        try:
            tag, data = salt.utils.event.SaltEvent.unpack(payload)
            log.debug("Incomming from peer %s %r", tag, data)
            if tag.startswith("cluster/peer/join-notify"):
                log.info(
                    "Cluster join notify from %s for %s",
                    data["peer_id"],
                    data["join_peer_id"],
                )
                peer_pub = (
                    pathlib.Path(self.opts["cluster_pki_dir"])
                    / "peers"
                    / f"{data['join_peer_id']}.pub"
                )
                with salt.utils.files.fopen(peer_pub, "w") as fp:
                    fp.write(data["pub"])
            elif tag.startswith("cluster/peer/join-reply"):
                log.info("Cluster join reply from %s", data["peer_id"])
                key = salt.crypt.PrivateKeyString(data["cluster_key"])
                key.write_private(self.opts["cluster_pki_dir"], "cluster")
                key.write_public(self.opts["cluster_pki_dir"], "cluster")
                for peer in data["peers"]:
                    log.error("Populate peer key %s", peer)
                    pub = (
                        pathlib.Path(self.opts["cluster_pki_dir"])
                        / "peers"
                        / f"{peer}.pub"
                    )
                    pub.write_text(data["peers"][peer])
                # XXX Initial pass just to get things working. This should be
                # able to be paged. We should also have the joining minion
                # request the keys it needs based on hashed values.
                for kind in data["minions"]:
                    for minion in (
                        pathlib.Path(self.opts["cluster_pki_dir"]) / kind
                    ).glob("*"):
                        if minion.name[:-4] not in data["minions"][kind]:
                            minion.unlink()
                    for minion in data["minions"][kind]:
                        log.error("Populate minion key %s", minion)
                        pub = (
                            pathlib.Path(self.opts["cluster_pki_dir"])
                            / kind
                            / f"{minion}"
                        )
                        pub.write_text(data["minions"][kind][minion])
                event = self._discover_event
                self._discover_event = None
                # Signal the main master process to start the rest of the
                # master service processeses.
                event.set()
            elif tag.startswith("cluster/peer/join"):

                payload = salt.payload.loads(data["payload"])

                pub, token = self._discover_candidates[payload["peer_id"]]

                if payload["pub"] != pub:
                    log.warning("Cluster join, peer public keys do not match")
                    return
                if payload["return_token"] != token:
                    log.warning("Cluster join, token does not not match")
                    return
                pubk = salt.crypt.PublicKeyString(payload["pub"])
                if not pubk.verify(data["payload"], data["sig"]):
                    log.warning("Cluster join signature invalid.")
                    return

                log.info("Cluster join from %s", payload["peer_id"])
                salted_secret = (
                    salt.crypt.PrivateKey.from_file(self.master_key.master_rsa_path)
                    .decrypt(payload["secret"])
                    .decode()
                )

                secret = salted_secret[len(token) :]

                if secret != (self.opts.get("cluster_secret") or ""):
                    log.warning("Cluster secret invalid.")
                    return

                log.info("Peer %s joined cluster", payload["peer_id"])
                salted_aes = (
                    salt.crypt.PrivateKey.from_file(self.master_key.master_rsa_path)
                    .decrypt(payload["key"])
                    .decode()
                )

                aes_key = salted_aes[len(token) :]

                # XXX needs safe join
                peer_pub = (
                    pathlib.Path(self.opts["cluster_pki_dir"])
                    / "peers"
                    / f"{payload['peer_id']}.pub"
                )
                # For statically-configured peers the pub key is already on
                # disk with restrictive perms. Only write when missing.
                if not peer_pub.exists():
                    with salt.utils.files.fopen(peer_pub, "w") as fp:
                        fp.write(payload["pub"])
                elif (
                    peer_pub.read_text(encoding="utf-8").strip()
                    != payload["pub"].strip()
                ):
                    log.warning(
                        "Cluster peer %s pub key on disk does not match the "
                        "key received during join; keeping disk copy.",
                        payload["peer_id"],
                    )

                self.cluster_peers.append(payload["peer_id"])
                self.pushers.append(self.pusher(payload["peer_id"]))
                self.auth_errors[payload["peer_id"]] = collections.deque()

                for pusher in self.pushers:
                    # XXX Send new peer id and public key to other nodes
                    # XXX This needs to be able to be validated by receiveing peers
                    # XXX Send other nodes pub (and aes?) keys to new node
                    crypticle = salt.crypt.Crypticle(
                        self.opts, salt.master.SMaster.secrets["aes"]["secret"].value
                    )
                    event_data = salt.utils.event.SaltEvent.pack(
                        salt.utils.event.tagify("join-notify", "peer", "cluster"),
                        crypticle.dumps(
                            {
                                "peer_id": self.opts["id"],
                                "join_peer_id": payload["peer_id"],
                                "pub": payload["pub"],
                                "aes": aes_key,
                            }
                        ),
                    )

                    # XXX gather tasks instead of looping
                    try:
                        await pusher.publish(event_data)
                    except Exception as exc:  # pylint: disable=broad-except
                        log.warning(
                            "Unable to publish join-notify to peer %s:%s: %s",
                            pusher.pull_host,
                            pusher.pull_port,
                            exc,
                        )

                # XXX Kick off minoins key repair

                self.send_aes_key_event()

                joiner_pub = salt.crypt.PublicKeyString(payload["pub"])
                token_bytes = (
                    payload["token"].encode()
                    if isinstance(payload["token"], str)
                    else payload["token"]
                )
                cluster_key_val = self.cluster_key() or ""
                cluster_key_bytes = (
                    cluster_key_val.encode()
                    if isinstance(cluster_key_val, str)
                    else cluster_key_val
                )
                aes_secret = salt.master.SMaster.secrets["aes"]["secret"].value
                if isinstance(aes_secret, str):
                    aes_secret = aes_secret.encode()
                # XXX No-shared-filesystem topology is not yet supported: the
                # join-reply payload still needs to carry the other peers'
                # public keys and the minion keys so a joiner without access
                # to a shared cluster_pki_dir can populate it from the wire.
                # The consumer at ``cluster/peer/join-reply`` also needs to
                # be reworked to unpack this signed envelope before that
                # path can be exercised.
                tosign = salt.payload.package(
                    {
                        "return_token": payload["token"],
                        "peer_id": self.opts["id"],
                        "cluster_key": joiner_pub.encrypt(
                            token_bytes + cluster_key_bytes
                        ),
                        "aes": joiner_pub.encrypt(token_bytes + aes_secret),
                    }
                )
                sig = salt.crypt.PrivateKeyString(self.private_key()).sign(tosign)
                event_data = salt.utils.event.SaltEvent.pack(
                    salt.utils.event.tagify("join-reply", "peer", "cluster"),
                    {
                        "sig": sig,
                        "payload": tosign,
                    },
                )
                await self.pusher(payload["peer_id"]).publish(event_data)
            elif tag.startswith("cluster/peer/discover-reply"):
                payload = salt.payload.loads(data["payload"])

                if not cluster_pub_matches_fingerprint(
                    self.opts, payload["cluster_pub"]
                ):
                    log.warning(
                        "cluster_pub fingerprint mismatch in discover-reply "
                        "from %s; rejecting",
                        payload.get("peer_id"),
                    )
                    return

                cluster_pub = salt.crypt.PublicKeyString(payload["cluster_pub"])
                if not cluster_pub.verify(data["payload"], data["sig"]):
                    log.warning("Invalid signature of cluster discover payload")
                    return

                # XXX First token created in different process
                # if payload.get("return_token", None) != self._discover_token:
                #    log.warning("Invalid token in discover reply %s != %s",
                #        payload.get("return_token", None), self._discover_token
                #    )
                #    return

                log.info("Cluster discover reply from %s", payload["peer_id"])
                key = salt.crypt.PublicKeyString(payload["pub"])
                self._discover_token = self.gen_token()
                tosign = salt.payload.package(
                    {
                        "return_token": payload["token"],
                        "token": self._discover_token,
                        "peer_id": self.opts["id"],
                        "secret": key.encrypt(
                            payload["token"].encode()
                            + (self.opts.get("cluster_secret") or "").encode()
                        ),
                        "key": key.encrypt(
                            payload["token"].encode()
                            + salt.master.SMaster.secrets["aes"]["secret"].value
                        ),
                        "pub": self.public_key(),
                    }
                )
                sig = salt.crypt.PrivateKeyString(self.private_key()).sign(tosign)
                self.cluster_peers.append(payload["peer_id"])
                event_data = salt.utils.event.SaltEvent.pack(
                    salt.utils.event.tagify("join", "peer", "cluster"),
                    {"sig": sig, "payload": tosign},
                )
                peer_pub = (
                    pathlib.Path(self.opts["cluster_pki_dir"])
                    / "peers"
                    / f"{payload['peer_id']}.pub"
                )
                # For statically-configured peers the pub key is already on
                # disk with restrictive perms (0400). Only write when it is
                # missing, otherwise verify the key on disk matches.
                if not peer_pub.exists():
                    with salt.utils.files.fopen(peer_pub, "w") as fp:
                        fp.write(payload["pub"])
                else:
                    existing = peer_pub.read_text(encoding="utf-8")
                    if existing.strip() != payload["pub"].strip():
                        log.warning(
                            "Cluster peer %s pub key on disk does not match "
                            "the key received during discovery; keeping disk "
                            "copy.",
                            payload["peer_id"],
                        )
                pusher = self.pusher(payload["peer_id"])
                self.pushers.append(pusher)
                try:
                    await pusher.publish(event_data)
                except Exception as exc:  # pylint: disable=broad-except
                    log.warning(
                        "Unable to publish join to peer %s:%s: %s",
                        pusher.pull_host,
                        pusher.pull_port,
                        exc,
                    )
            elif tag.startswith("cluster/peer/discover"):
                payload = salt.payload.loads(data["payload"])
                peer_key = salt.crypt.PublicKeyString(payload["pub"])
                if not peer_key.verify(data["payload"], data["sig"]):
                    log.warning("Invalid signature of cluster discover payload")
                    return
                log.info("Cluster discovery from %s", payload["peer_id"])
                token = self.gen_token()
                # Store this peer as a candidate.
                # XXX Add timestamp so we can clean up old candidates
                self._discover_candidates[payload["peer_id"]] = (payload["pub"], token)
                tosign = salt.payload.package(
                    {
                        "return_token": payload["token"],
                        "token": token,
                        "peer_id": self.opts["id"],
                        "pub": self.public_key(),
                        "cluster_pub": self.cluster_public_key(),
                    }
                )
                key = salt.crypt.PrivateKeyString(self.cluster_key())
                sig = key.sign(tosign)
                _ = salt.payload.package(
                    {
                        "sig": sig,
                        "payload": tosign,
                    }
                )
                event_data = salt.utils.event.SaltEvent.pack(
                    salt.utils.event.tagify("discover-reply", "peer", "cluster"),
                    {"sig": sig, "payload": tosign},
                )
                await self.pusher(payload["peer_id"]).publish(event_data)
            elif tag.startswith("cluster/peer"):
                peer = data["peer_id"]
                if peer == self.opts["id"]:
                    log.debug("Skip our own cluster peer event %s", tag)
                    return
                aes = data["peers"][self.opts["id"]]["aes"]
                sig = data["peers"][self.opts["id"]]["sig"]
                key_str = self.master_key.master_key.decrypt(
                    aes, algorithm="OAEP-SHA224"
                )
                digest = salt.utils.stringutils.to_bytes(
                    hashlib.sha256(key_str).hexdigest()
                )
                key = self.master_key.fetch(f"peers/{peer}.pub")
                m_digest = key.decrypt(sig)
                if m_digest != digest:
                    log.error("Invalid aes signature from peer: %s", peer)
                    return
                log.info("Received new AES key from peer %s", peer)
                if peer in self.peer_keys:
                    if self.peer_keys[peer] != key_str:
                        self.peer_keys[peer] = key_str
                        self.send_aes_key_event()
                        while self.auth_errors[peer]:
                            key, data = self.auth_errors[peer].popleft()
                            peer_id, parsed_tag = self.parse_cluster_tag(tag)
                            try:
                                event_data = self.extract_cluster_event(peer_id, data)
                            except salt.exceptions.AuthenticationError:
                                log.error(
                                    "Event from peer failed authentication: %s", peer_id
                                )
                            else:
                                await self.transport.publish_payload(
                                    salt.utils.event.SaltEvent.pack(
                                        parsed_tag, event_data
                                    )
                                )
                else:
                    self.peer_keys[peer] = key_str
                    self.send_aes_key_event()
                    while self.auth_errors[peer]:
                        key, data = self.auth_errors[peer].popleft()
                        peer_id, parsed_tag = self.parse_cluster_tag(tag)
                        try:
                            event_data = self.extract_cluster_event(peer_id, data)
                        except salt.exceptions.AuthenticationError:
                            log.error(
                                "Event from peer failed authentication: %s", peer_id
                            )
                        else:
                            await self.transport.publish_payload(
                                salt.utils.event.SaltEvent.pack(parsed_tag, event_data)
                            )
            elif tag.startswith("cluster/event"):
                peer_id, parsed_tag = self.parse_cluster_tag(tag)
                try:
                    event_data = self.extract_cluster_event(peer_id, data)
                except salt.exceptions.AuthenticationError:
                    self.auth_errors[peer_id].append((tag, data))
                else:
                    await self.transport.publish_payload(
                        salt.utils.event.SaltEvent.pack(parsed_tag, event_data)
                    )
            else:
                log.error("This cluster tag not valid %s", tag)
        except Exception:  # pylint: disable=broad-except
            log.critical("Unhandled error while polling master events", exc_info=True)
            return None

    def parse_cluster_tag(self, tag):
        peer_id = tag.replace("cluster/event/", "").split("/")[0]
        stripped_tag = tag.replace(f"cluster/event/{peer_id}/", "")
        return peer_id, stripped_tag

    def extract_cluster_event(self, peer_id, data):
        if peer_id in self.peer_keys:
            crypticle = _get_crypticle(self.opts, self.peer_keys[peer_id])
            event_data = crypticle.loads(data)["event_payload"]
            # __peer_id can be used to know if this event came from a
            # different master.
            event_data["__peer_id"] = peer_id
            return event_data
        raise salt.exceptions.AuthenticationError("Peer aes key not available")

    async def publish_payload(self, load, *args):
        tag, data = salt.utils.event.SaltEvent.unpack(load)
        # log.warning("Event %s %s %r", len(self.pushers), tag, data)
        tasks = []
        if not tag.startswith("cluster/peer"):
            tasks = [
                asyncio.create_task(
                    self.transport.publish_payload(load), name=self.opts["id"]
                )
            ]
        for pusher in self.pushers:
            log.info("Publish event to peer %s:%s", pusher.pull_host, pusher.pull_port)
            if tag.startswith("cluster/peer"):
                # log.info("Send %s %r", tag, load)
                tasks.append(
                    asyncio.create_task(pusher.publish(load), name=pusher.pull_host)
                )
                continue
            crypticle = _get_crypticle(
                self.opts, salt.master.SMaster.secrets["aes"]["secret"].value
            )
            load = {"event_payload": data}
            event_data = salt.utils.event.SaltEvent.pack(
                salt.utils.event.tagify(tag, self.opts["id"], "cluster/event"),
                crypticle.dumps(load),
            )
            tasks.append(asyncio.create_task(pusher.publish(event_data)))
        await asyncio.gather(*tasks, return_exceptions=True)
        for task in tasks:
            try:
                task.result()
            # XXX This error is transport specific and should be something else
            except tornado.iostream.StreamClosedError:
                if task.get_name() == self.opts["id"]:
                    log.error("Unable to forward event to local ipc bus")
                else:
                    peer = task.get_name()
                    log.warning(
                        "Unable to forward event to cluster peer %s; "
                        "resetting pusher for reconnect",
                        peer,
                    )
                    # Reset the broken pub_sock so the next publish attempt
                    # triggers a fresh TCP connection rather than reusing a
                    # dead stream.
                    for pusher in self.pushers:
                        if pusher.pull_host == peer and pusher.pub_sock is not None:
                            try:
                                pusher.pub_sock.close()
                            except Exception:  # pylint: disable=broad-except
                                pass
                            pusher.pub_sock = None
                    # Schedule an AES-key re-announcement so the peer
                    # learns our key after it reconnects.
                    self.io_loop.call_later(2.0, self.send_aes_key_event)
            except Exception as exc:  # pylint: disable=broad-except
                log.error(
                    "Unhandled error sending task %s", task.get_name(), exc_info=True
                )
