# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
"""Registry compile operations.

This module provides the core logic for the `registry compile` command,
which scans a registry bucket, determines the "latest" GA version for each
connector, ensures `latest/` directories are up-to-date, and writes
global registry JSON index files plus per-connector version indexes.

High-level algorithm
--------------------
1. **Scan** -- glob for `metadata.yaml` files to discover all
   (connector, version) tuples.  No file *contents* are downloaded.
2. **Yank set** -- glob for `version-yank.yml` markers to build an
   exclusion set.
3. **Compute latest** -- for each connector, pick the highest GA semver
   that is not yanked and not a pre-release.
4. **Fast-check latest/** -- glob for `version=*` marker files inside
   `latest/` directories.  Compare with the computed latest.  Only
   connectors whose marker disagrees (or is missing) need a resync.
5. **Resync stale latest/** -- delete the old `latest/` directory,
   ensure a `version=<x.y.z>` marker exists in the versioned
   directory, then recursively copy the versioned directory to
   `latest/`.
6. **Write index files**:
   a. Global `registries/v0/cloud_registry.json` and
      `registries/v0/oss_registry.json` (backwards-compatible).
   b. Per-connector `versions.json` (new).

When `with_secrets_mask=True`, the compile step also regenerates
`registries/v0/specs_secrets_mask.yaml` by scanning all connector specs
for properties marked `airbyte_secret: true`.
"""

from __future__ import annotations

import contextlib
import copy
import json
import logging
import os
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from typing import Any

import dpath.util
import gcsfs
import yaml
from packaging.version import InvalidVersion, Version

from airbyte_ops_mcp.registry._constants import (
    METADATA_FOLDER,
)

logger = logging.getLogger(__name__)

# Regex for the `version=<semver>` marker filename inside `latest/`
_VERSION_MARKER_RE = re.compile(r"^version=(.+)$")

# Registry index output path (relative to bucket root)
_REGISTRIES_PREFIX = "registries/v0"

# Specs secrets mask filename
_SPECS_SECRETS_MASK_FILENAME = "specs_secrets_mask.yaml"


# ---------------------------------------------------------------------------
# Result model
# ---------------------------------------------------------------------------


@dataclass
class CompileResult:
    """Result of a registry compile operation."""

    target: str
    connectors_scanned: int = 0
    versions_found: int = 0
    yanked_versions: int = 0
    latest_updated: int = 0
    latest_already_current: int = 0
    cloud_registry_entries: int = 0
    oss_registry_entries: int = 0
    version_indexes_written: int = 0
    specs_secrets_mask_properties: int = 0
    errors: list[str] = field(default_factory=list)
    dry_run: bool = False

    @property
    def status(self) -> str:
        if self.dry_run:
            return "dry-run"
        if self.errors:
            return "completed-with-errors"
        return "success"

    def summary(self) -> str:
        return (
            f"[{self.status}] Scanned {self.connectors_scanned} connectors, "
            f"{self.versions_found} versions ({self.yanked_versions} yanked). "
            f"Latest updated: {self.latest_updated}, "
            f"already current: {self.latest_already_current}. "
            f"Registry entries: cloud={self.cloud_registry_entries}, "
            f"oss={self.oss_registry_entries}. "
            f"Version indexes: {self.version_indexes_written}. "
            f"Specs secrets mask: {self.specs_secrets_mask_properties} properties. "
            f"Errors: {len(self.errors)}."
        )


@dataclass
class PurgeLatestResult:
    """Result of a purge-latest operation."""

    target: str
    connectors_found: int = 0
    latest_dirs_deleted: int = 0
    errors: list[str] = field(default_factory=list)
    dry_run: bool = False

    @property
    def status(self) -> str:
        if self.dry_run:
            return "dry-run"
        if self.errors:
            return "completed-with-errors"
        return "success"

    def summary(self) -> str:
        return (
            f"[{self.status}] Found {self.connectors_found} connectors, "
            f"deleted {self.latest_dirs_deleted} latest/ directories. "
            f"Errors: {len(self.errors)}."
        )


@dataclass
class ConnectorVersionInfo:
    """Lightweight info about a single published version."""

    version: str
    yanked: bool = False
    is_latest: bool = False
    release_stage: str | None = None
    support_level: str | None = None


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _log_progress(msg: str, *args: object) -> None:
    """Log a progress message to both the logger and stderr."""
    logger.info(msg, *args)
    formatted = msg % args if args else msg
    print(formatted, file=sys.stderr, flush=True)


def _get_gcs_credentials_token() -> dict[str, str]:
    """Get GCS credentials as a token dict for gcsfs."""
    gcs_creds = os.environ.get("GCS_CREDENTIALS")
    if not gcs_creds:
        raise ValueError(
            "GCS_CREDENTIALS environment variable is required for GCS operations."
        )
    return json.loads(gcs_creds)


def _is_ga_version(version_str: str) -> bool:
    """Return True if the version string is a GA (non-prerelease) semver."""
    try:
        v = Version(version_str)
    except InvalidVersion:
        return False
    return not v.is_prerelease and not v.is_devrelease


def _parse_version(version_str: str) -> Version | None:
    """Parse a version string, returning None if it is not valid semver."""
    try:
        return Version(version_str)
    except InvalidVersion:
        return None


def _extract_connector_version(
    path: str, bucket: str, prefix: str = ""
) -> tuple[str, str] | None:
    """Extract (connector_name, version) from a GCS path.

    Expected path format:
        `<bucket>/[<prefix>/]metadata/airbyte/<connector>/<version>/metadata.yaml`

    Args:
        path: Full GCS path returned by glob.
        bucket: Bucket name.
        prefix: Optional path prefix (e.g. `"devin-test/"` or `""`).  Must
            include trailing `/` if non-empty.

    Returns None if the path does not match.
    """
    expected_prefix = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/"
    if not path.startswith(expected_prefix):
        return None
    remainder = path[len(expected_prefix) :]
    parts = remainder.split("/")
    if len(parts) < 2:
        return None
    connector = parts[0]
    version = parts[1]
    # Skip non-version dirs like "latest", "release_candidate"
    if version in ("latest", "release_candidate"):
        return None
    return connector, version


# ---------------------------------------------------------------------------
# Step 1 + 2: Scan versions and yank markers via glob
# ---------------------------------------------------------------------------


def _scan_versions_and_yanks(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector_name: list[str] | None = None,
) -> tuple[dict[str, list[str]], set[tuple[str, str]]]:
    """Scan the registry for all (connector, version) tuples and yank markers.

    Uses efficient glob patterns -- no file contents are downloaded.

    Args:
        fs: Authenticated GCSFileSystem.
        bucket: Bucket name (with optional prefix already applied).
        prefix: Path prefix within the bucket (e.g. "" or "aj-test100/").
        connector_name: If provided, only scan these connector names.

    Returns:
        Tuple of:
        - dict mapping connector_name -> list of version strings
        - set of (connector_name, version) tuples that are yanked
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"

    # Step 1: glob for metadata.yaml to discover all versions
    if connector_name:
        metadata_paths: list[str] = []
        for name in connector_name:
            pattern = f"{base}/{name}/*/metadata.yaml"
            metadata_paths.extend(fs.glob(pattern))
    else:
        pattern = f"{base}/*/*/metadata.yaml"
        metadata_paths = fs.glob(pattern)

    _log_progress("Glob found %d metadata.yaml files", len(metadata_paths))

    connector_versions: dict[str, list[str]] = {}
    for path in metadata_paths:
        parsed = _extract_connector_version(path, bucket, prefix)
        if parsed is None:
            continue
        connector, version = parsed
        connector_versions.setdefault(connector, []).append(version)

    # Step 2: glob for version-yank.yml markers
    if connector_name:
        yank_paths: list[str] = []
        for name in connector_name:
            yank_pattern = f"{base}/{name}/*/version-yank.yml"
            yank_paths.extend(fs.glob(yank_pattern))
    else:
        yank_pattern = f"{base}/*/*/version-yank.yml"
        yank_paths = fs.glob(yank_pattern)

    _log_progress("Glob found %d version-yank.yml markers", len(yank_paths))

    yanked: set[tuple[str, str]] = set()
    for path in yank_paths:
        # Path: bucket/[prefix/]metadata/airbyte/<connector>/<version>/version-yank.yml
        parts = path.split("/")
        # Find "metadata" index, then airbyte/<connector>/<version>
        try:
            meta_idx = parts.index("metadata")
            connector = parts[meta_idx + 2]
            version = parts[meta_idx + 3]
            yanked.add((connector, version))
        except (ValueError, IndexError):
            logger.warning("Could not parse yank path: %s", path)

    return connector_versions, yanked


# ---------------------------------------------------------------------------
# Step 3: Compute latest GA version per connector
# ---------------------------------------------------------------------------


def _compute_latest_versions(
    connector_versions: dict[str, list[str]],
    yanked: set[tuple[str, str]],
) -> dict[str, str]:
    """For each connector, determine the highest GA semver that is not yanked.

    Returns:
        dict mapping connector_name -> latest version string.
        Connectors with no eligible GA versions are omitted.
    """
    latest: dict[str, str] = {}
    for connector, versions in connector_versions.items():
        candidates: list[tuple[Version, str]] = []
        for v_str in versions:
            if (connector, v_str) in yanked:
                continue
            if not _is_ga_version(v_str):
                continue
            parsed = _parse_version(v_str)
            if parsed is not None:
                candidates.append((parsed, v_str))
        if candidates:
            candidates.sort(reverse=True)
            latest[connector] = candidates[0][1]
        else:
            logger.warning(
                "No eligible GA version found for %s (%d total versions, all yanked or pre-release).",
                connector,
                len(versions),
            )
    return latest


# ---------------------------------------------------------------------------
# Step 4: Check version markers in latest/ directories
# ---------------------------------------------------------------------------


def _scan_latest_markers(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector_name: list[str] | None = None,
) -> dict[str, str]:
    """Glob for `version=*` marker files in `latest/` directories.

    Returns:
        dict mapping connector_name -> current marker version string.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"

    if connector_name:
        marker_paths: list[str] = []
        for name in connector_name:
            marker_pattern = f"{base}/{name}/latest/version=*"
            marker_paths.extend(fs.glob(marker_pattern))
    else:
        marker_pattern = f"{base}/*/latest/version=*"
        marker_paths = fs.glob(marker_pattern)

    _log_progress("Glob found %d version marker files in latest/", len(marker_paths))

    markers: dict[str, str] = {}
    for path in marker_paths:
        filename = path.split("/")[-1]
        match = _VERSION_MARKER_RE.match(filename)
        if not match:
            continue
        version_str = match.group(1)
        # Extract connector name from path
        parts = path.split("/")
        try:
            meta_idx = parts.index("metadata")
            connector = parts[meta_idx + 2]
            markers[connector] = version_str
        except (ValueError, IndexError):
            logger.warning("Could not parse marker path: %s", path)

    return markers


# ---------------------------------------------------------------------------
# Step 5: Resync stale latest/ directories
# ---------------------------------------------------------------------------


def _delete_latest_dir(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
) -> None:
    """Delete a connector's `latest/` directory recursively.

    This is the shared primitive used by both compile (step 5, before
    copying the new version) and delete-dev-latest (standalone purge).

    Silently succeeds if the directory does not exist.
    """
    latest_path = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}/latest"
    with contextlib.suppress(FileNotFoundError):
        fs.rm(latest_path, recursive=True)


def _ensure_version_marker(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
    version: str,
) -> None:
    """Ensure a `version=<semver>` marker file exists in the versioned dir.

    If the marker already exists this is a no-op.  Otherwise a zero-byte
    file is created so that it will be included in the recursive copy to
    `latest/`.
    """
    marker_path = (
        f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"
        f"/{version}/version={version}"
    )
    if not fs.exists(marker_path):
        with fs.open(marker_path, "wb") as f:
            f.write(b"")
        _log_progress("  Created missing marker in versioned dir: version=%s", version)


def _sync_latest_dir(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
    version: str,
    dry_run: bool = False,
) -> None:
    """Replace `latest/` with a recursive copy of the versioned directory.

    Algorithm:
    1. Ensure a `version=<semver>` marker exists in the versioned dir
       (backfill; once the publish pipeline writes it, this becomes a no-op).
    2. Delete `latest/` recursively via `_delete_latest_dir()`.
    3. Recursively copy the entire versioned directory to `latest/`.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"
    source_dir = f"{base}/{version}"
    dest_dir = f"{base}/latest"

    if dry_run:
        _log_progress(
            "  [DRY RUN] Would sync %s/ → %s/",
            source_dir,
            dest_dir,
        )
        return

    _log_progress("  Syncing %s/ → %s/", source_dir, dest_dir)

    # 1. Ensure marker exists in the versioned directory.
    _ensure_version_marker(fs, bucket, prefix, connector, version)

    # 2. Delete existing latest/ directory.
    _delete_latest_dir(fs, bucket, prefix, connector)

    # 3. Recursive copy from versioned dir → latest/.
    fs.copy(source_dir, dest_dir, recursive=True)

    _log_progress("  Synced %s/ → %s/", source_dir, dest_dir)


def _apply_overrides_to_latest_entry(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
    version: str,
) -> None:
    """Post-process `latest/` registry entries after syncing from a versioned dir.

    When the compile step copies files from `{version}/` to `latest/`, the
    embedded `cloud.json` and `oss.json` still contain version-specific
    values.  This function:

    1. Applies `registryOverrides` from the metadata so that the `latest/`
       entry reflects the overridden values (e.g. a pinned `dockerImageTag`).
    2. When an override pins `dockerImageTag` to a different version, copies
       version-sensitive fields (`spec`, `packageInfo`) from the pinned
       version's entry so that `latest/` is fully consistent.
    3. Updates `generated.source_file_info.metadata_file_path` to reference
       the `latest/` path instead of the versioned path.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"
    latest_dir = f"{base}/latest"

    # Read metadata.yaml from latest/ to get registry overrides.
    metadata_path = f"{latest_dir}/metadata.yaml"
    try:
        with fs.open(metadata_path, "r") as f:
            raw_metadata = yaml.safe_load(f)
    except FileNotFoundError:
        logger.warning(
            "No metadata.yaml in latest/ for %s, skipping override application",
            connector,
        )
        return
    except Exception as exc:
        logger.warning("Failed to read metadata for %s: %s", connector, exc)
        return

    metadata_data: dict[str, Any] = raw_metadata.get("data", {})
    registry_overrides = metadata_data.get("registryOverrides", {})

    latest_metadata_file_path = (
        f"{METADATA_FOLDER}/airbyte/{connector}/latest/metadata.yaml"
    )

    for registry_type in ("cloud", "oss"):
        entry_path = f"{latest_dir}/{registry_type}.json"
        try:
            if not fs.exists(entry_path):
                continue
            with fs.open(entry_path, "r") as f:
                entry = json.load(f)
        except Exception as exc:
            logger.warning(
                "Failed to read %s.json for %s: %s", registry_type, connector, exc
            )
            continue

        modified = False

        # --- Apply registry overrides (skip_docker_image_tag=False for latest) ---
        overrides = copy.deepcopy(registry_overrides.get(registry_type, {}))
        overrides.pop("enabled", None)
        overrides = {k: v for k, v in overrides.items() if v is not None}

        if overrides:
            entry.update(overrides)
            modified = True

        # --- When dockerImageTag is overridden to a different version, copy
        #     version-sensitive fields from the pinned version's entry. ---
        pinned_tag = overrides.get("dockerImageTag")
        if pinned_tag and pinned_tag != version:
            pinned_entry_path = f"{base}/{pinned_tag}/{registry_type}.json"
            try:
                if fs.exists(pinned_entry_path):
                    with fs.open(pinned_entry_path, "r") as fp:
                        pinned_entry = json.load(fp)
                    for field_name in ("spec", "packageInfo"):
                        if field_name in pinned_entry:
                            entry[field_name] = pinned_entry[field_name]
                            modified = True
                else:
                    logger.warning(
                        "Pinned version %s/%s.json not found for %s",
                        pinned_tag,
                        registry_type,
                        connector,
                    )
            except Exception as exc:
                logger.warning(
                    "Failed to read pinned entry %s/%s.json for %s: %s",
                    pinned_tag,
                    registry_type,
                    connector,
                    exc,
                )

        # --- Update metadata_file_path to point to latest/ ---
        generated = entry.get("generated")
        if isinstance(generated, dict):
            source_file_info = generated.get("source_file_info")
            if isinstance(source_file_info, dict):
                current_path = source_file_info.get("metadata_file_path")
                if current_path != latest_metadata_file_path:
                    source_file_info["metadata_file_path"] = latest_metadata_file_path
                    modified = True

        if modified:
            content = json.dumps(entry, indent=2, sort_keys=True) + "\n"
            with fs.open(entry_path, "w") as fout:
                fout.write(content)


# ---------------------------------------------------------------------------
# Step 6a: Compile global registry JSONs
# ---------------------------------------------------------------------------


def _compile_global_registry(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    latest_versions: dict[str, str],
    registry_type: str,
) -> list[dict[str, Any]]:
    """Read cloud.json or oss.json from each connector's latest/ dir.

    Reads are parallelised with up to ``_COMPILE_READ_MAX_WORKERS`` threads
    to avoid the ~150 s serial penalty on a full registry (~1 250 files).

    Args:
        registry_type: "cloud" or "oss".

    Returns:
        List of registry entry dicts (deterministic order, sorted by connector name).
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"
    sorted_connectors = sorted(latest_versions)

    def _read_one(connector: str) -> dict[str, Any] | None:
        json_path = f"{base}/{connector}/latest/{registry_type}.json"
        try:
            if not fs.exists(json_path):
                return None
            with fs.open(json_path, "r") as f:
                return json.load(f)  # type: ignore[no-any-return]
        except Exception as exc:
            logger.warning(
                "Failed to read %s for %s: %s", registry_type, connector, exc
            )
            return None

    entries: list[dict[str, Any]] = []
    with ThreadPoolExecutor(max_workers=_COMPILE_READ_MAX_WORKERS) as pool:
        results = list(pool.map(_read_one, sorted_connectors))
    for result in results:
        if result is not None:
            entries.append(result)

    return entries


def _build_global_registry_json(entries: list[dict[str, Any]]) -> dict[str, Any]:
    """Assemble the global registry JSON from a list of connector entries.

    Splits entries into sources and destinations based on the presence of
    `sourceDefinitionId` or `destinationDefinitionId`.
    """
    sources = []
    destinations = []
    for entry in entries:
        if "sourceDefinitionId" in entry:
            sources.append(entry)
        elif "destinationDefinitionId" in entry:
            destinations.append(entry)
        else:
            # Fallback: guess from dockerRepository
            docker_repo = entry.get("dockerRepository", "")
            if "destination" in docker_repo:
                destinations.append(entry)
            else:
                sources.append(entry)

    return {"sources": sources, "destinations": destinations}


# ---------------------------------------------------------------------------
# Step 6c: Specs secrets mask
# ---------------------------------------------------------------------------


def _extract_secret_property_names(
    entries: list[dict[str, Any]],
) -> set[str]:
    """Extract property names marked as `airbyte_secret` from connector specs.

    Walks the `spec.connectionSpecification.properties` tree of each entry
    and collects the leaf property name for every node that has
    `airbyte_secret: true`.

    Args:
        entries: List of registry entry dicts (from cloud or oss registries).

    Returns:
        A set of property names that are marked as secrets.
    """
    secret_names: set[str] = set()
    for entry in entries:
        spec = entry.get("spec", {})
        conn_spec = spec.get("connectionSpecification", {})
        properties = conn_spec.get("properties")
        if not properties:
            continue
        for type_path, _ in dpath.util.search(properties, "**/type", yielded=True):
            absolute_path = f"/{type_path}"
            if "/" in type_path:
                property_path, _ = absolute_path.rsplit(sep="/", maxsplit=1)
            else:
                property_path = absolute_path
            try:
                property_definition = dpath.util.get(properties, property_path)
            except KeyError:
                continue
            if isinstance(property_definition, dict) and property_definition.get(
                "airbyte_secret", False
            ):
                secret_names.add(property_path.split("/")[-1])
    return secret_names


# ---------------------------------------------------------------------------
# Step 6b: Per-connector version index
# ---------------------------------------------------------------------------


def _build_version_index(
    connector: str,
    versions: list[str],
    yanked: set[tuple[str, str]],
    latest_version: str | None,
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
) -> dict[str, Any]:
    """Build the per-connector versions.json content.

    For each version, reads `metadata.yaml` to extract `releaseStage`
    and `supportLevel` (only for the latest version to keep scanning fast).
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"

    version_entries: list[dict[str, Any]] = []
    for v_str in sorted(
        versions, key=lambda s: _parse_version(s) or Version("0"), reverse=True
    ):
        is_yanked = (connector, v_str) in yanked
        is_latest = v_str == latest_version

        entry: dict[str, Any] = {
            "version": v_str,
            "yanked": is_yanked,
            "is_latest": is_latest,
        }

        # For the latest version, enrich with metadata fields
        if is_latest:
            try:
                meta_path = f"{base}/{v_str}/metadata.yaml"
                with fs.open(meta_path, "r") as f:
                    raw = yaml.safe_load(f)
                data = raw.get("data", {})
                entry["release_stage"] = data.get("releaseStage")
                entry["support_level"] = data.get("supportLevel")
            except Exception as exc:
                logger.warning(
                    "Failed to read metadata for %s@%s: %s", connector, v_str, exc
                )

        version_entries.append(entry)

    definition_id = None
    if latest_version:
        # Try to get the definition ID from the latest version's metadata
        try:
            meta_path = f"{base}/{latest_version}/metadata.yaml"
            with fs.open(meta_path, "r") as f:
                raw = yaml.safe_load(f)
            data = raw.get("data", {})
            definition_id = data.get("definitionId")
        except Exception:
            pass

    result: dict[str, Any] = {
        "connector": connector,
        "versions": version_entries,
    }
    if definition_id:
        result["definition_id"] = definition_id

    return result


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


_PURGE_LATEST_MAX_WORKERS = 20
"""DoP for purge-latest bulk deletes."""

_COMPILE_SYNC_MAX_WORKERS = 80
"""DoP for Step 5 — resync stale latest/ directories."""

_COMPILE_READ_MAX_WORKERS = 20
"""DoP for Step 6a — read cloud.json / oss.json from latest/ dirs."""

_COMPILE_WRITE_MAX_WORKERS = 80
"""DoP for Step 6b — write per-connector versions.json indexes."""


def purge_latest_dirs(
    bucket_name: str,
    prefix: str = "",
    connector_name: list[str] | None = None,
    dry_run: bool = False,
) -> PurgeLatestResult:
    """Delete all `latest/` directories from the registry store.

    Discovers connector directories via glob, then deletes each
    `latest/` subdirectory in parallel using a thread pool.

    Args:
        bucket_name: GCS bucket name.
        prefix: Optional path prefix within the bucket.
        connector_name: If provided, only purge these connectors.
        dry_run: If True, report what would be done without deleting.

    Returns:
        A `PurgeLatestResult` describing what was done.
    """
    prefix = f"{prefix}/" if prefix else ""
    target_label = f"{bucket_name}/{prefix}" if prefix else bucket_name
    result = PurgeLatestResult(target=target_label, dry_run=dry_run)

    token = _get_gcs_credentials_token()
    fs = gcsfs.GCSFileSystem(token=token)

    base = f"{bucket_name}/{prefix}{METADATA_FOLDER}/airbyte"

    # Discover latest/ dirs by listing connector directories that contain
    # a `latest/` subdirectory.
    _log_progress("Discovering latest/ directories...")
    base_with_slash = f"{base}/"
    if connector_name:
        # Check each requested connector for a latest/ dir
        seen: set[str] = set()
        connectors_with_latest: list[str] = []
        for name in connector_name:
            if name in seen:
                continue
            latest_path = f"{base}/{name}/latest"
            if fs.exists(latest_path):
                connectors_with_latest.append(name)
                seen.add(name)
    else:
        # Glob for all connectors, then filter to those with latest/
        all_connector_dirs = fs.glob(f"{base}/*/latest")
        seen = set()
        connectors_with_latest = []
        for path in all_connector_dirs:
            # Strip the known base prefix and take the first component
            if not path.startswith(base_with_slash):
                logger.warning("Could not parse latest path: %s", path)
                continue
            relative = path[len(base_with_slash) :]
            connector = relative.split("/")[0]
            if connector and connector not in seen:
                connectors_with_latest.append(connector)
                seen.add(connector)

    result.connectors_found = len(connectors_with_latest)
    _log_progress(
        "Found %d connectors with latest/ directories",
        result.connectors_found,
    )

    if not connectors_with_latest:
        _log_progress("Nothing to purge.")
        _log_progress(result.summary())
        return result

    if dry_run:
        for connector in sorted(connectors_with_latest):
            _log_progress("  [DRY RUN] Would delete %s/latest/", connector)
        result.latest_dirs_deleted = len(connectors_with_latest)
        _log_progress(result.summary())
        return result

    # Delete latest/ dirs in parallel using the shared helper.
    def _delete_one(connector: str) -> str | None:
        """Delete a single connector's latest/ dir. Returns error string or None."""
        try:
            _delete_latest_dir(fs, bucket_name, prefix, connector)
            return None
        except Exception as exc:
            return f"Failed to delete latest/ for {connector}: {exc}"

    _log_progress(
        "Deleting %d latest/ directories (max_workers=%d)...",
        len(connectors_with_latest),
        _PURGE_LATEST_MAX_WORKERS,
    )

    with ThreadPoolExecutor(max_workers=_PURGE_LATEST_MAX_WORKERS) as pool:
        futures = {
            pool.submit(_delete_one, c): c for c in sorted(connectors_with_latest)
        }
        for i, future in enumerate(as_completed(futures), 1):
            connector = futures[future]
            error = future.result()
            if error:
                logger.error(error)
                result.errors.append(error)
            else:
                result.latest_dirs_deleted += 1
            if i % 100 == 0:
                _log_progress("  Deleted %d / %d...", i, len(connectors_with_latest))

    _log_progress(result.summary())
    return result


def compile_registry(
    bucket_name: str,
    prefix: str = "",
    connector_name: list[str] | None = None,
    dry_run: bool = False,
    with_secrets_mask: bool = False,
) -> CompileResult:
    """Compile the registry: sync latest/ dirs and write index files.

    Steps:
        1. Glob for all `metadata.yaml` to discover (connector, version) pairs.
        2. Glob for `version-yank.yml` to build the yanked set.
        3. Compute the latest GA semver per connector.
        4. Glob for `version=*` markers in `latest/` dirs for a fast check.
        5. Delete stale `latest/` dirs and recursively copy the versioned dir.
        6. Write global registry JSONs and per-connector `versions.json`.
        6c. (Optional) Regenerate `specs_secrets_mask.yaml`.

    Args:
        bucket_name: GCS bucket name to compile from/to.
        prefix: Optional path prefix within the bucket.
        connector_name: If provided, only compile these connectors.
        dry_run: If True, report what would be done without writing.
        with_secrets_mask: If True, regenerate `specs_secrets_mask.yaml`.

    Returns:
        A `CompileResult` describing what was done.
    """
    prefix = f"{prefix}/" if prefix else ""

    target_label = f"{bucket_name}/{prefix}" if prefix else bucket_name
    result = CompileResult(target=target_label, dry_run=dry_run)

    token = _get_gcs_credentials_token()
    fs = gcsfs.GCSFileSystem(token=token)

    # --- Step 1 + 2: Scan versions and yanks ---
    _log_progress("Step 1-2: Scanning versions and yank markers...")
    connector_versions, yanked = _scan_versions_and_yanks(
        fs, bucket_name, prefix, connector_name
    )
    result.connectors_scanned = len(connector_versions)
    result.versions_found = sum(len(v) for v in connector_versions.values())
    result.yanked_versions = len(yanked)
    _log_progress(
        "  Found %d connectors, %d versions, %d yanked",
        result.connectors_scanned,
        result.versions_found,
        result.yanked_versions,
    )

    # --- Step 3: Compute latest ---
    _log_progress("Step 3: Computing latest GA version per connector...")
    latest_versions = _compute_latest_versions(connector_versions, yanked)
    _log_progress("  Computed latest for %d connectors", len(latest_versions))

    # --- Step 4: Check existing latest markers ---
    _log_progress("Step 4: Checking existing latest/ markers...")
    existing_markers = _scan_latest_markers(fs, bucket_name, prefix, connector_name)
    _log_progress("  Found %d existing markers", len(existing_markers))

    stale_connectors: list[str] = []
    for connector, expected_version in latest_versions.items():
        current_marker = existing_markers.get(connector)
        if current_marker == expected_version:
            result.latest_already_current += 1
        else:
            stale_connectors.append(connector)

    _log_progress(
        "  %d connectors need latest/ update, %d already current",
        len(stale_connectors),
        result.latest_already_current,
    )

    # --- Step 5: Resync stale latest/ dirs (parallel) ---
    if stale_connectors:
        _log_progress(
            "Step 5: Syncing %d stale latest/ directories (max_workers=%d)...",
            len(stale_connectors),
            _COMPILE_SYNC_MAX_WORKERS,
        )

        def _sync_one_connector(connector: str) -> None:
            """Sync a single connector's latest/ dir."""
            version = latest_versions[connector]
            _sync_latest_dir(fs, bucket_name, prefix, connector, version, dry_run)
            if not dry_run:
                _apply_overrides_to_latest_entry(
                    fs, bucket_name, prefix, connector, version
                )

        sorted_stale = sorted(stale_connectors)
        with ThreadPoolExecutor(max_workers=_COMPILE_SYNC_MAX_WORKERS) as pool:
            futures = {pool.submit(_sync_one_connector, c): c for c in sorted_stale}
            for i, future in enumerate(as_completed(futures), 1):
                connector = futures[future]
                try:
                    future.result()
                    result.latest_updated += 1
                except Exception as exc:
                    error_msg = f"Failed to sync latest/ for {connector}: {exc}"
                    logger.error(error_msg)
                    result.errors.append(error_msg)
                    # Delete the (possibly partial) latest/ dir so the next
                    # compile retries this connector from scratch.
                    try:
                        _delete_latest_dir(fs, bucket_name, prefix, connector)
                        logger.info(
                            "Cleaned up partial latest/ for %s after failure",
                            connector,
                        )
                    except Exception as cleanup_exc:
                        logger.warning(
                            "Could not clean up latest/ for %s: %s",
                            connector,
                            cleanup_exc,
                        )
                if i % 100 == 0:
                    _log_progress("  Synced %d / %d...", i, len(sorted_stale))
    else:
        _log_progress("Step 5: All latest/ directories are current, nothing to sync.")

    # --- Step 6a: Compile global registry JSONs ---
    _log_progress("Step 6a: Compiling global registry JSON files...")
    all_registry_entries: list[dict[str, Any]] = []  # collected for Step 6c
    for registry_type in ("cloud", "oss"):
        entries = _compile_global_registry(
            fs, bucket_name, prefix, latest_versions, registry_type
        )
        all_registry_entries.extend(entries)
        registry_json = _build_global_registry_json(entries)

        output_path = (
            f"{bucket_name}/{prefix}{_REGISTRIES_PREFIX}/{registry_type}_registry.json"
        )
        entry_count = len(registry_json["sources"]) + len(registry_json["destinations"])

        if registry_type == "cloud":
            result.cloud_registry_entries = entry_count
        else:
            result.oss_registry_entries = entry_count

        if dry_run:
            _log_progress(
                "  [DRY RUN] Would write %s_registry.json (%d entries)",
                registry_type,
                entry_count,
            )
        else:
            content = json.dumps(registry_json, indent=2, sort_keys=True) + "\n"
            with fs.open(output_path, "w") as f:
                f.write(content)
            _log_progress(
                "  Wrote %s_registry.json (%d entries)",
                registry_type,
                entry_count,
            )

    # --- Step 6b: Per-connector version indexes (parallel) ---
    _log_progress(
        "Step 6b: Writing per-connector version indexes (max_workers=%d)...",
        _COMPILE_WRITE_MAX_WORKERS,
    )
    base = f"{bucket_name}/{prefix}{METADATA_FOLDER}/airbyte"
    sorted_connectors = sorted(connector_versions)

    def _write_one_version_index(connector: str) -> None:
        """Build and write a single connector's versions.json."""
        versions = connector_versions[connector]
        latest_v = latest_versions.get(connector)
        index = _build_version_index(
            connector, versions, yanked, latest_v, fs, bucket_name, prefix
        )
        index_path = f"{base}/{connector}/versions.json"
        if dry_run:
            _log_progress(
                "  [DRY RUN] Would write %s/versions.json (%d versions)",
                connector,
                len(versions),
            )
        else:
            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
            with fs.open(index_path, "w") as f:
                f.write(content)

    with ThreadPoolExecutor(max_workers=_COMPILE_WRITE_MAX_WORKERS) as pool:
        futures = {
            pool.submit(_write_one_version_index, c): c for c in sorted_connectors
        }
        for i, future in enumerate(as_completed(futures), 1):
            connector = futures[future]
            try:
                future.result()
                result.version_indexes_written += 1
            except Exception as exc:
                error_msg = f"Failed to write versions.json for {connector}: {exc}"
                logger.error(error_msg)
                result.errors.append(error_msg)
            if i % 100 == 0:
                _log_progress(
                    "  Wrote %d / %d version indexes...", i, len(sorted_connectors)
                )

    # --- Step 6c: Specs secrets mask (optional) ---
    if with_secrets_mask:
        if connector_name:
            _log_progress(
                "Step 6c: SKIPPED — --with-secrets-mask requires all connectors "
                "but --connector-name is set. Run without --connector-name "
                "to regenerate the full secrets mask."
            )
        else:
            _log_progress("Step 6c: Generating specs secrets mask...")
            # Reuse entries collected during Step 6a to avoid redundant GCS reads.
            secret_names = _extract_secret_property_names(all_registry_entries)
            sorted_names = sorted(secret_names)
            result.specs_secrets_mask_properties = len(sorted_names)
            mask_content = yaml.dump(
                {"properties": sorted_names}, default_flow_style=False
            )
            mask_path = f"{bucket_name}/{prefix}{_REGISTRIES_PREFIX}/{_SPECS_SECRETS_MASK_FILENAME}"

            _log_progress(
                "  Found %d secret properties: %s",
                len(sorted_names),
                ", ".join(sorted_names),
            )

            if dry_run:
                _log_progress(
                    "  [DRY RUN] Would write %s",
                    _SPECS_SECRETS_MASK_FILENAME,
                )
            else:
                with fs.open(mask_path, "w") as f:
                    f.write(mask_content)
                _log_progress(
                    "  Wrote %s",
                    _SPECS_SECRETS_MASK_FILENAME,
                )

    _log_progress(result.summary())
    return result
