"""
Run reprepro commands

.. error:: **reprepro** (bookworm): Sometimes hangs on keyring package in testsuite (blocking all) (:debbug:`1056380`)

  Reproduce: ``./devel profile _all updatetestall``.

  This was introduced with the fix for (:debbug:`1050321`) in ``5.3.1-1+deb12u1``. Unfortunately, this
  version is still [Apr 2024] in stable/bookworm. Fixed for trixie+.

.. attention:: **reprepro**: Fails with ``debian/`` being a symlink in Debian native packages (:debbug:`768046`)

  In such a case, builds will be fine, but reprepro will not be able to install the package; you will only be able to
  see reprepro's error "No section and no priority for" in the ``daemon.log``.

  For the moment, just avoid such a setup (which is imho not desirable anyway). However, as it's a legal setup afaik it
  should work after all.

.. attention:: **reprepro**: Don't switch to ``reprepro 5.4`` yet (:debbug:`1017983`)

  As of *June 2023* ``5.4.2``, this is still in experimental.

  There are still db migration problems to be fixed. Furthermore, I regularly experience lockups in full testsuite runs.

.. attention:: **reprepro** (``dist.py`` setup workaround): Can't handle Ubuntu's ``ddeb`` (:debbug:`730572`)

.. note:: **reprepro**: Multiple versions of packages in one distribution

  This is not really a problem, but a uncommon situation that may lead to confusion.

  Generally, reprepro does allow exactly only one version of a package in a distribution; the only exception is when
  installed in *different components* (f.e., ``main`` vs. ``non-free``).

  This usually happens when the 'Section' changes in the corresponding 'debian/control' file of the source package, or if
  packages were installed manually using "-C" with reprepro.

  Check with the "ls" command if this is the case, i.e., s.th. like:

  .. code:: bash

    mini-buildd-api ls ENDPOINT my-package

  you may see multiple entries for one distribution with different components.

  mini-buildd handles this gracefully; the :apicall:`remove`, the :apicall:`migrate` and the :apicall:`port` all include
  an optional 'version' parameter to be able to select a specific version.

  In the automated rollback handling, all versions of a source package are shifted.
"""

import glob
import logging
import os
import shutil
import threading

import debian.debian_support

from mini_buildd import call, changes, config, dist, util

LOG = logging.getLogger(__name__)

_LOCKS = {}

#: For doc/tests only
LSBYCOMPONENT_EXAMPLE = """
mbd-test-cpp | 20220530080821~test~SID+1 |           sid-test-unstable | main | source
mbd-test-cpp | 20220530070229~test~SID+1 | sid-test-unstable-rollback0 | main | source
"""


class Ls(dict):
    """
    >>> ls = Ls("mbd-test-cpp", LSBYCOMPONENT_EXAMPLE)

    >>> ls.rollbacks()
    {'sid-test-unstable-rollback0': {'version': '20220530070229~test~SID+1', 'component': 'main', 'architecture': 'source'}}
    >>> ls.filter(ls, diststr="sid-test-unstable")
    {'sid-test-unstable': {'rollbacks': {'sid-test-unstable-rollback0': {'version': '20220530070229~test~SID+1', 'component': 'main', 'architecture': 'source'}}, 'version': '20220530080821~test~SID+1', 'component': 'main', 'architecture': 'source'}}
    >>> ls.filter(ls.rollbacks(), version="20220530070229~test~SID+1")
    {'sid-test-unstable-rollback0': {'version': '20220530070229~test~SID+1', 'component': 'main', 'architecture': 'source'}}
    """

    def __init__(self, source, ls_output, codenames=None, version=None, min_version=None):
        super().__init__()
        self.source = source

        self.codenames = set()
        for item in ls_output.split("\n"):
            if item:
                # reprepro lsbycomponent format: "<source> | <version> | <distribution> | <component> | <architecture>"
                split = item.split("|")
                _dist = dist.Dist(split[2].strip())
                values = {
                    "version": split[1].strip(),
                    "component": split[3].strip(),
                    "architecture": split[4].strip(),
                }
                if (codenames is None or _dist.codename in codenames) and (not version or values["version"] == version) and (not min_version or debian.debian_support.Version(values["version"]) >= debian.debian_support.Version(min_version)):
                    self.codenames.add(dist.Codename(_dist.codename))
                    save_to = self.setdefault(_dist.get(rollback=False), {"rollbacks": {}})
                    if _dist.is_rollback:
                        save_to["rollbacks"][_dist.get()] = values
                    else:
                        # Need to set individual items here to not possibly overwrite 'rollback'
                        for k, v in values.items():
                            save_to[k] = v

        self.codenames = sorted(self.codenames)

    def enrich(self):
        for diststr in self:
            ls = self[diststr]

            repository, distribution, suite = util.models().parse_diststr(diststr)

            if ls.get("version") is not None:
                ls["dsc_path"] = repository.mbd_dsc_pool_path(self.source, ls["version"], raise_exception=False)

                # Add changes files for convenience only - you can get to the changes files via 'events for this version' link
                events_path = config.ROUTES["events"].path
                ls["changes_paths"] = [events_path.removeprefix(p) for p in glob.glob(events_path.join(self.source, ls.get("version"), "**", "*_source.changes"), recursive=True)]

            ls["uploadable"] = suite.uploadable
            if suite.migrates_to:
                migrates_to = repository.mbd_get_diststr(distribution, suite.migrates_to)
                ls["migrates_to"] = migrates_to
                if ls.get("version") is not None:
                    ls["is_migrated"] = bool(self.filter(diststr=migrates_to, version=ls["version"]))

            for rollback_diststr in ls["rollbacks"]:
                rollback_ls = ls["rollbacks"][rollback_diststr]
                if rollback_ls.get("version") is not None:
                    rollback_ls["dsc_path"] = repository.mbd_dsc_pool_path(self.source, rollback_ls["version"], raise_exception=False)

    def rollbacks(self, diststr=None):
        result = {}
        for _d, _v in self.items():
            if diststr is None or _d == diststr:
                for d, v in _v["rollbacks"].items():
                    result[d] = v
        return result

    def filter(self, ls=None, diststr=None, version=None, raise_if_found="", raise_if_not_found=""):
        if ls is None:
            ls = self  # Non-rollback search

        result = {}
        for ls_diststr, ls_values in ls.items():
            if ls_values.get("version") is not None and (diststr is None or ls_diststr == diststr) and (version is None or ls_values.get("version") == version):
                result[ls_diststr] = ls_values
        if result and raise_if_found:
            raise util.HTTPBadRequest(f"{raise_if_found}: Source '{self.source}_{version or 'any'}' found in '{diststr or 'any'}' distribution")
        if not result and raise_if_not_found:
            raise util.HTTPBadRequest(f"{raise_if_not_found}: Source '{self.source}_{version or 'any'}' not found in '{diststr or 'any'}' distribution")
        return result

    def filter_rollbacks(self, diststr=None, **kwargs):
        return self.filter(self.rollbacks(diststr), **kwargs)

    def filter_all(self, **kwargs):
        return self.filter({**self, **self.rollbacks()}, **kwargs)

    def changes(self, diststr, version, ls=None, **kwargs):
        values = self.filter(ls=ls, diststr=diststr, version=version, raise_if_not_found="Internal")[diststr]
        return changes.Changes({"Distribution": diststr, "Source": self.source, "Version": values["version"], "Architecture": "source", "Component": values["component"]}, **kwargs)


class Reprepro():
    r"""
    Abstraction to reprepro repository commands

    *Locking*

    This implicitly provides a locking mechanism to avoid
    parallel calls to the same repository from mini-buildd
    itself. This rules out any failed call due to reprepro
    locking errors in the first place.

    For the case that someone else is using reprepro
    manually, we also always run it with '--waitforlock'.

    *Ignoring 'unusedarch' check*

    Known broken use case is linux' 'make deb-pkg' up to version 4.13.

    linux' native 'make deb-pkg' is the recommended and documented way to
    produce custom kernels on Debian systems.

    Up to linux version 4.13 (see [l1]_, [l2]_), this would also produce
    firmware packages, flagged "arch=all" in the control file, but
    actually producing "arch=any" firmware \*.deb. The changes file
    produced however would still list "all" in the Architecture field,
    making the reprepro "unsusedarch" check fail (and thusly, installation
    on mini-buildd will fail).

    While this is definitely a bug in 'make deb-pkg' (and also not an
    issue 4.14 onwards or when you use it w/o producing a firmware
    package), the check is documented as "safe to ignore" in reprepro, so
    I think we should allow these cases to work.

    .. [l1] https://github.com/torvalds/linux/commit/cc18abbe449aafc013831a8e0440afc336ae1cba
    .. [l2] https://github.com/torvalds/linux/commit/5620a0d1aacd554ebebcff373e31107bb1ef7769
    """

    def __init__(self, basedir):
        self._basedir = basedir
        self._cmd = ["reprepro", "--verbose", "--waitforlock", "10", "--ignore", "unusedarch", "--basedir", f"{basedir}"]
        self._lock = _LOCKS.setdefault(self._basedir, threading.Lock())

    def __str__(self):
        return f"Reprepro repository at {self._basedir}"

    def _call(self, args):
        return call.Call(self._cmd + args).check().log().stdout

    def _call_locked(self, args):
        with self._lock:
            return self._call(args)

    @classmethod
    def clean_exportable_indices(cls, dist_dir):
        LOG.info("Reprepro: Cleaning exportable indices from %s...", dist_dir)
        for item in os.listdir(dist_dir):
            if item != "snapshots":
                rm_path = os.path.join(dist_dir, item)
                LOG.debug("PURGING: %s", rm_path)
                if os.path.isdir(rm_path) and not os.path.islink(rm_path):
                    shutil.rmtree(rm_path, ignore_errors=True)
                else:
                    os.remove(rm_path)

    def reindex(self, distributions):
        """
        (Re-)index repository

        Remove all files from repository that can be re-indexed via
        "reprepro export", and remove all files that would not (i.e.,
        stale files from removed distributions).

        Historically, this would just remove 'dists/' completely. With
        the support of reprepro snapshots however, extra care must be
        taken that valid snapshots are *not removed* (as they are not
        reprepro-exportable) and stale snapshots (from removed
        distributions) are *properly unreferenced* in the database
        prior to removing. See doc for 'gen_snapshot' in reprepro's
        manual.

        """
        LOG.info("(Re)indexing %s", self)
        with self._lock:
            dists_dir = os.path.join(self._basedir, "dists")
            os.makedirs(dists_dir, exist_ok=True)

            for dist_str in os.listdir(dists_dir):
                dist_dir = os.path.join(dists_dir, dist_str)
                self.clean_exportable_indices(dist_dir)

                _dist = dist.Dist(dist_str)
                if _dist.get(rollback=False) not in distributions:
                    LOG.debug("Dist has been removed from repo: %s", dist_dir)
                    for s in self.get_snapshots(dist_str):
                        self._del_snapshot(dist_str, s)  # Call w/o locking!
                    LOG.debug("PURGING: %s", dist_dir)
                    shutil.rmtree(dist_dir, ignore_errors=True)

            # Update reprepro dbs, and delete any packages no longer in dists.
            self._call(["--delete", "clearvanished"])

            # Finally, rebuild all exportable indices
            self._call(["export"])

    def check(self):
        return self._call_locked(["check"])

    def ls(self, source, **filter_options):
        return Ls(source, self._call_locked(["--type", "dsc", "lsbycomponent", source]), **filter_options)

    def migrate(self, package, src_distribution, dst_distribution, version=None):
        return self._call_locked(["copysrc", dst_distribution, src_distribution, package] + ([version] if version else []))

    def remove(self, package, distribution, version=None):
        return self._call_locked(["removesrc", distribution, package] + ([version] if version else []))

    def install(self, _changes, distribution):
        return self._call_locked(["include", distribution, _changes])

    def install_dsc(self, dsc, distribution):
        return self._call_locked(["includedsc", distribution, dsc])

    #
    # Reprepro Snapshots
    #
    def snapshots_dir(self, distribution):
        if not distribution:
            raise util.HTTPBadRequest("Empty reprepro snapshots distribution name")
        return os.path.join(self._basedir, "dists", distribution, "snapshots")

    def snapshot_dir(self, distribution, name):
        sd = self.snapshots_dir(distribution)
        if not name:
            raise util.HTTPBadRequest("Empty reprepro snapshot name")
        return os.path.join(sd, name)

    def get_snapshots(self, distribution, prefix=""):
        sd = self.snapshots_dir(distribution)
        return [s for s in os.listdir(sd) if s.startswith(prefix)] if os.path.exists(sd) else []

    def gen_snapshot(self, distribution, name):
        sd = self.snapshot_dir(distribution, name)
        if os.path.exists(sd):
            raise util.HTTPBadRequest(f"Reprepro snapshot '{name}' for '{distribution}' already exists")

        with self._lock:
            self._call(["gensnapshot", distribution, name])

    def _del_snapshot(self, distribution, name):
        sd = self.snapshot_dir(distribution, name)
        if not os.path.exists(sd):
            raise util.HTTPBadRequest(f"Reprepro snapshot '{name}' for '{distribution}' does not exist")

        LOG.info("Deleting reprepro snapshot '%s'...", sd)
        self._call(["unreferencesnapshot", distribution, name])
        self._call(["--delete", "clearvanished"])
        self._call(["deleteunreferenced"])
        shutil.rmtree(sd, ignore_errors=True)

    def del_snapshot(self, distribution, name):
        with self._lock:
            self._del_snapshot(distribution, name)
