"""
API arguments and calls

>>> Setup.from_command_line("--save --update all --archives-from-proxy --sources bullseye --sources-from-origin-with-lts Debian --chroots-from-sources --repositories test/test").command_line()
'--save --update all --archives-from-proxy --sources bullseye --sources-from-origin-with-lts Debian --repositories test/test --chroots-from-sources'

>>> Setup.from_sloppy_args(save=True, archives_from_proxy=True, archives_from_apt_sources=False).command_line()
'--save --archives-from-proxy'
"""

import abc
import os
import sys
import inspect
import collections
import logging
import argparse
import re
import glob
import shlex
import hashlib
import json
import threading
from contextlib import closing

import mini_buildd.misc
import mini_buildd.net
import mini_buildd.changes
import mini_buildd.package

from mini_buildd.dist import SETUP, CODENAMES, Dist
from mini_buildd.values import OnOff, get_value
from mini_buildd.values import Name as Value

LOG = logging.getLogger(__name__)


class Argument():
    """
    Generic Argument Class

    ``value()`` always provides a non-``None`` value of the specific
    type, either the ``default`` or a ``given`` value.

    ``strvalue()`` always provides a non-``None`` ``str`` value.

    The ``default`` value is given in the constructor. For
    server-specific defaults, this may be function -- then the default
    value will be computed only at run time on the server.

    A ``given`` value can be provided via special ``set()`` method:
      * Empty ``str``, ``list`` or false ``bool`` will yield ``None``.
      * Non-empty ``str`` will be converted to the specific type.
      * Other given values will be used as is.

    ============  ============ ============ =========== =================
    Type          value() type svalue() ex. HTML GET    argparse
    ============  ============ ============ =========== =================
    Str           str          "string"     key=string  --key "string"
    Url           str          "http://.."  key=string  --key "string"
    MultilineStr  str          "long"       key=string  --key "string"
    Choice        str          "c0"         key=string  --key "string"
    Int           int          "17"         key=string  --key "int"
    Bool          bool         "True"       key=True    --key
    List          list of str  "v0,v1,.."   key=v0,v1.. --key "v0" "v1"..
    ============  ============ ============ =========== =================
    """

    #: Validate that values are actually of that type
    VALUE_TYPE = str

    #: Magic string value to use as value when a default callable on the server should be used.
    SERVER_DEFAULT = "<server_default>"

    def __init__(self, id_list, doc="Undocumented", default=None, choices=None, header=None):
        """
        :param id_list: List like '['--with-rollbacks', '-R']' for option or '['distributions']' for positionals; 1st entry always denotes the id.

        >>> Argument(["--long-id", "-s"]).identity
        'long_id'
        >>> Argument(["posi-tional"]).identity
        'posi_tional'
        """
        # Identifiers && doc
        self.id_list = id_list
        self.doc = doc
        self.is_positional = not id_list[0].startswith("--")
        # identity: 1st of id_list with leading '--' removed and hyphens turned to snake case
        self.identity = mini_buildd.misc.Snake(id_list[0][0 if self.is_positional else 2:]).from_kebab()

        # Values
        self._default = default
        self.given = None

        # Choices helper
        self._choices = choices
        self.header = header

    def __str__(self):
        """Be sure not to use value() here"""
        return f"{self.identity}: given={self.given}, _default={self._default}"

    @classmethod
    def s2v(cls, str_value):
        """Convert string to value"""
        return cls.VALUE_TYPE(str_value) if str_value else None

    @classmethod
    def v2s(cls, value):
        """Convert value to string"""
        return "" if value is None else str(value)

    def required(self):
        return self._default is None

    def needs_value(self):
        """If user input is no_value"""
        return self.given is None and self.required()

    def choices(self):
        return [] if self._choices is None else get_value(self._choices)

    def default(self):
        return get_value(self._default)

    def strdefault(self):
        return self.v2s(self.default())

    def value(self):
        if self.needs_value():
            raise mini_buildd.HTTPBadRequest(f"Missing required argument: {self}")
        return self.default() if self.given is None else self.given

    def strvalue(self):
        return self.v2s(self.value())

    def strgiven(self):
        return self.v2s(self.given)

    def icommand_line_given(self):
        yield self.strgiven()

    def set(self, given):
        if given == self.SERVER_DEFAULT:
            self.given = None
        elif isinstance(given, str):
            self.given = self.s2v(given)
        elif isinstance(given, (list, bool)):
            self.given = given if given else None
        elif isinstance(given, self.VALUE_TYPE):
            self.given = given
        else:
            raise Exception(f"API argument '{self.identity}': Invalid value {given} (type given {type(given)}, needs {self.VALUE_TYPE})")

    def argparse_kvsargs(self):
        """Python 'argparse' support"""
        kvsargs = {"help": self.doc}
        if self._default is not None:
            if isinstance(self._default, Value):
                kvsargs["default"] = self.SERVER_DEFAULT
                kvsargs["help"] += f"\nServer Default: {self._default.name}"
            else:
                kvsargs["default"] = self._default
        return kvsargs


class StrArgument(Argument):
    HTML_TYPE = "text"

    def argparse_kvsargs(self):
        return {**super().argparse_kvsargs(), **{"action": "store"}}


class UrlArgument(StrArgument):
    HTML_TYPE = "url"


class MultilineStrArgument(StrArgument):
    HTML_TYPE = "textarea"


class ChoiceArgument(Argument):
    HTML_TYPE = "select"

    def value(self):
        value = super().value()
        choices = self.choices()
        if value not in choices:
            raise mini_buildd.HTTPBadRequest(f"{self.identity}: Wrong choice argument: '{value}' not in {choices}")
        return value

    def argparse_kvsargs(self):
        kvsargs = super().argparse_kvsargs()
        kvsargs["choices"] = self._choices
        if self.is_positional and self._default is not None:
            kvsargs["nargs"] = "?"  # Allow positional args to be actually optional, if we have a default.
        return kvsargs


class IntArgument(StrArgument):
    VALUE_TYPE = int
    HTML_TYPE = "number"

    def argparse_kvsargs(self):
        return {**super().argparse_kvsargs(), **{"type": int}}


class BoolArgument(ChoiceArgument):
    VALUE_TYPE = bool
    HTML_TYPE = "checkbox"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, choices=[True, False], default=False, **kwargs)

    @classmethod
    def s2v(cls, str_value):
        return str_value == str(True)

    def icommand_line_given(self):
        """Empty generator -- bools are just command line options like ``--with-foo``"""
        yield from []

    def argparse_kvsargs(self):
        kvsargs = Argument.argparse_kvsargs(self)
        kvsargs["action"] = "store_true"
        return kvsargs


class ListArgument(StrArgument):
    VALUE_TYPE = list
    SEPARATOR = ","

    @classmethod
    def s2v(cls, str_value):
        return str_value.split(cls.SEPARATOR) if str_value else None

    @classmethod
    def v2s(cls, value):
        return "" if value is None else cls.SEPARATOR.join(value)

    def icommand_line_given(self):
        if self.given is not None:
            yield from self.given

    def argparse_kvsargs(self):
        kvsargs = super().argparse_kvsargs()
        kvsargs["nargs"] = "+"
        if self._choices is not None and not isinstance(self._choices, Value):
            kvsargs["choices"] = self._choices
        kvsargs["metavar"] = mini_buildd.misc.singularize(self.identity.upper())
        return kvsargs


#: Specialized argument classes
class Repository(StrArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(id_list, choices=Value.ALL_REPOSITORIES, doc="Repository identity (like 'test', the default sandbox repo identity)", **kwargs)

    def object(self):
        return mini_buildd.mdls().repository.get(self.value())


class Repositories(ListArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(
            id_list,
            choices=Value.ALL_REPOSITORIES,
            doc="Repository identities (like 'test', the default sandbox repo identity)",
            **kwargs,
        )

    def objects(self):
        return mini_buildd.mdls().repository.Repository.objects.filter(identity__in=self.value())


class Codenames(ListArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(
            id_list,
            choices=Value.ALL_CODENAMES,
            doc="Codenames (like 'buster', 'bullseye')",
            **kwargs,
        )


def diststr2repository(diststr):
    return mini_buildd.mdls().repository.get(Dist(diststr).repository)


class Distribution(StrArgument):
    def __init__(self, id_list, choices=Value.ACTIVE_DISTRIBUTIONS, extra_doc="", **kwargs):
        super().__init__(id_list, choices=choices, doc="Distribution name (<codename>-<id>-<suite>[-rollback<n>])" + extra_doc, **kwargs)

    def dist(self):
        return mini_buildd.dist.Dist(self.value())


class Distributions(ListArgument):
    def __init__(self, id_list, choices=Value.ACTIVE_DISTRIBUTIONS, **kwargs):
        super().__init__(id_list, choices=choices, doc="Distribution names", **kwargs)


class Source(StrArgument):
    def __init__(self, id_list, extra_doc="", **kwargs):
        super().__init__(id_list, choices=Value.LAST_SOURCES, doc="Source package name" + extra_doc, **kwargs)


class UploadOptions(StrArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(id_list,
                         default="lintian-mode=ignore",
                         doc=("List of upload options, separated by '|':\n"
                              "\n"
                              f"{mini_buildd.changes.Upload.Options.usage()}"
                              ),
                         **kwargs)


class Output(ChoiceArgument):
    """Meta API call option 'output'"""

    def __init__(self):
        super().__init__(["output"], choices=["html", "json"], default="html",
                         doc=("desired result format\n"
                              "\n"
                              "html: Return result as HTML (popup with JS, else full HTML page)\n"
                              "json: Return raw json result\n"
                              ))


class Rollbacks(IntArgument):
    def __init__(self, id_list, extra_doc="", **kwargs):
        super().__init__(id_list, default=0, doc="Select (maximal) <n> rollback sources (``0``: no rollbacks, ``< 0``: all rollbacks)" + extra_doc, **kwargs)

    def range(self, suite):
        """Get valid range for this suite"""
        rollbacks = self.value()
        if rollbacks < 0:
            rollbacks = suite.rollback
        return range(0, min(rollbacks, suite.rollback))


class Call():
    AUTH = mini_buildd.config.AUTH_NONE
    NEEDS_RUNNING_DAEMON = False
    CONFIRM = False
    CUSTOM_HTML_SNIPPET = None

    _LOCK = threading.Lock()  # Optional use

    @classmethod
    def name(cls):
        return mini_buildd.misc.Snake(cls.__name__).from_camel()

    @classmethod
    def doc(cls):
        return inspect.getdoc(cls)

    @classmethod
    def doc_title(cls):
        return cls.doc().partition("\n")[0]

    @classmethod
    def doc_body(cls):
        return cls.doc().partition("\n")[2][1:]

    # Category support -- purely for structuring documentation.
    CATEGORIES = ["Consumer", "Developer", "Administrator"]

    @classmethod
    def category(cls):
        if cls.AUTH in [mini_buildd.config.AUTH_NONE, mini_buildd.config.AUTH_LOGIN]:
            return cls.CATEGORIES[0]
        if cls.AUTH in [mini_buildd.config.AUTH_STAFF]:
            return cls.CATEGORIES[1]
        return cls.CATEGORIES[2]

    @classmethod
    def uri(cls):
        return mini_buildd.config.URIS["api"]["view"].join(cls.name() + "/")

    @classmethod
    def iarguments(cls):
        yield from []

    @classmethod
    def from_sloppy_args(cls, **kwargs):
        """Construct ignoring any unknown arguments given"""
        keys = [a.identity for a in cls.iarguments()]
        return cls(**{key: value for key, value in kwargs.items() if key in keys})

    def set_args(self, **kwargs):
        LOG.debug(f"{self.name()}: Given args: {kwargs}")
        for key, value in kwargs.items():
            if key not in self.args:
                raise mini_buildd.HTTPBadRequest(f"API call '{self.name()}': Unknown argument '{key}'")
            self.args[key].set(value)
            LOG.debug(f"{self.name()}: Adding given arg: {key}={value}")

        for key in (key for key in self.args if key in kwargs):
            self.args[key].set(kwargs.get(key))

    def __init__(self, **kwargs):
        self.args = {arg.identity: arg for arg in self.iarguments()}
        self.set_args(**kwargs)
        self.request = None
        self.result = {}  # json result

    @classmethod
    def from_command_line(cls, command_line):
        return cls(**cls.parse_command_line(command_line))

    def set_request(self, request):
        self.request = request

    @classmethod
    def get_plain(cls, result, force_json=False):
        """Get ``str`` result (non-``str`` results get json pretty-formatted)"""
        return mini_buildd.misc.json_pretty(result) if (force_json or not isinstance(result, str)) else result

    def plain(self, force_json=False):
        return self.get_plain(self.result, force_json=force_json)

    def json_pretty(self):
        """For (arg-less) use in templates only"""
        return self.plain(force_json=True)

    @classmethod
    def parse_command_line(cls, command_line):
        class RaisingArgumentParser(argparse.ArgumentParser):
            """Make argparse raise only (not exit) on error. See https://bugs.python.org/issue41255"""

            def error(self, message):
                raise Exception(message)

        parser = RaisingArgumentParser()
        for argument in cls.iarguments():
            parser.add_argument(*argument.id_list, **argument.argparse_kvsargs())
        return vars(parser.parse_args(args=shlex.split(command_line)))

    def icommand_line(self, full=False, with_user=False, user=None, exclude=None):
        _exclude = [] if exclude is None else exclude

        if full:
            yield "mini-buildd-api"
            yield self.name()
            yield mini_buildd.http_endpoint().geturl(with_user=with_user, user=user)

        for arg in (a for a in self.args.values() if a.identity not in _exclude and a.given is not None):
            if arg.is_positional:
                yield from arg.icommand_line_given()
            else:
                yield arg.id_list[0]
                yield from arg.icommand_line_given()

    def command_line(self, full=False, with_user=False, user=None, exclude=None):
        return mini_buildd.PyCompat.shlex_join(self.icommand_line(full=full, with_user=with_user, user=user, exclude=exclude))

    def command_line_full(self):
        user = self.request.user if self.AUTH is not mini_buildd.config.AUTH_NONE and self.request is not None and self.request.user.is_authenticated else None
        return self.command_line(full=True, with_user=user is not None, user=user)

    def http_args(self, with_confirm=False, with_output=None):
        http_args = {}
        for arg in self.args.values():
            if arg.given is not None:
                http_args[arg.identity] = arg.strgiven()

        if with_confirm:
            http_args["confirm"] = self.name()
        if with_output is not None:
            http_args["output"] = with_output

        return http_args

    def url(self):
        return mini_buildd.http_endpoint().geturl(path=mini_buildd.config.URIS["api"]["view"].join(self.name() + "/"), query=self.http_args(with_output="html"))

    @abc.abstractmethod
    def run(self):
        pass


def _pimpdoc(result_doc=None):
    def pimp(cls):
        indent = "    "  # Be sure to keep the same indent like a top level class doc (4 chars) for all lines, so inspect.getdoc() will properly unindent.
        if result_doc:
            cls.__doc__ += "\n" + "\n".join([f"{indent}{line}" for line in result_doc]) + "\n"
        cls.__doc__ += f"\n{indent}Authorization: {cls.AUTH}\n"
        return cls
    return pimp


#: Option shortcuts (use as mixin)
class _Admin:
    AUTH = mini_buildd.config.AUTH_ADMIN


class _Staff:
    AUTH = mini_buildd.config.AUTH_STAFF


class _Login:
    AUTH = mini_buildd.config.AUTH_LOGIN


class _Running:
    NEEDS_RUNNING_DAEMON = True


class _Confirm:
    CONFIRM = True


#: API calls
@_pimpdoc()
class Status(Call):
    """
    Get the status of this instance

    JSON Result:

    * version     : mini-buildd's version.
    * identity    : Instance identity.
    * url         : Instance URL (HTTP).
    * incoming_url: Incoming URL (currently FTP).
    * load        : Instance's (0 =< load <= 1). If negative, the instance is not powered on.
    * chroots     : Active chroots.
    * remotes     : Active or auto-reactivatable remotes.
    * [repositories: Simplified structural representation of all repositories]
    """

    @classmethod
    def iarguments(cls):
        yield BoolArgument(["--with-repositories", "-R"], doc="Also list all repositories (may be longish)")

    def run(self):
        self.result = {
            "version": mini_buildd.__version__,
            "identity": mini_buildd.get_daemon().model.identity,
            "url": mini_buildd.http_endpoint().geturl(),
            "incoming_url": mini_buildd.get_daemon().model.mbd_get_ftp_endpoint().geturl(),
            "load": mini_buildd.get_daemon().builder.load() if mini_buildd.get_daemon().is_alive() else -1.0,
            "chroots": [c.mbd_key() for c in mini_buildd.mdls().chroot.Chroot.mbd_get_active()],
            "remotes": [r.mbd_url() for r in mini_buildd.mdls().gnupg.Remote.mbd_get_active_or_auto_reactivate()],
        }
        if self.args["with_repositories"].value():
            self.result["repositories"] = {r.identity: r.mbd_json() for r in mini_buildd.mdls().repository.Repository.objects.all()}


@_pimpdoc()
class PubKey(Call):
    """
    Get public key

    Get ASCII-armored GnuPG public key of this instance.

    Used to sign the apt repositories (apt key) and for authorization across instances.
    """

    def run(self):
        self.result = mini_buildd.get_daemon().gnupg.pub_key


@_pimpdoc()
class DputConf(Call):
    """
    Get recommended dput config snippet

    Usually, this is for integration in your personal ~/.dput.cf.

    .. tip:: Can I add a custom dput config snippet?

      Just put your custom snippet in ``~mini-buildd/etc/dput.cf``. If that file exists, it will be added to the output of :apicall:`dput_conf`.
    """

    def run(self):
        self.result = mini_buildd.get_daemon().model.mbd_get_dput_conf()

        # Optionally add local custom config
        try:
            with mini_buildd.fopen(mini_buildd.config.ROUTES["etc"].path.join("dput.cf")) as extra_conf:
                self.result += extra_conf.read()
        except FileNotFoundError:
            pass


@_pimpdoc()
class SourcesList(Call):
    """
    Get sources.list (apt lines)

    Usually, this output is put to a file like '/etc/sources.list.d/mini-buildd-xyz.list'.
    """

    @classmethod
    def iarguments(cls):
        yield Codenames(["--codenames", "-C"], default=Value.ALL_CODENAMES)
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield ListArgument(["--suites", "-S"], default=Value.ALL_SUITES, choices=Value.ALL_SUITES, doc="Suite names (like 'stable', 'unstable')")
        yield ListArgument(["--types", "-T"], default=["deb"], choices=["deb", "deb-src"], doc="Types of apt lines")
        yield StrArgument(["--options", "-O"], default="", doc="Apt line options ('deb[-src] [<options>] ...'). See 'man 5 source.list'")
        yield Rollbacks(["--rollbacks", "-r"])
        yield StrArgument(["--snapshot", "-P"], default="", doc="Select a repository snapshot")
        yield StrArgument(["--mirror", "-M"], default="", doc="URL of a mirror to use (instead of mini-buildd's native URL)")
        yield BoolArgument(["--with-extra", "-X"], doc="Also list extra sources needed")
        yield BoolArgument(["--with-comment", "-D"], doc="Add comment line above apt line")

    def run(self):
        sources_list = mini_buildd.files.SourcesList()

        for r in self.args["repositories"].objects():
            for d in r.distributions.all().filter(base_source__codename__in=self.args["codenames"].value()):
                if self.args["with_extra"].value():
                    for e in d.extra_sources.all():
                        sources_list.append(e.source.mbd_get_apt_line())

                for s in r.layout.suiteoption_set.filter(suite__name__in=self.args["suites"].value()):
                    sources_list.append(r.mbd_get_apt_line(d, s, snapshot=self.args["snapshot"].value()))
                    for rollback in self.args["rollbacks"].range(s):
                        sources_list.append(r.mbd_get_apt_line(d, s, snapshot=self.args["snapshot"].value(), rollback=rollback))

        self.result = sources_list.get(self.args["types"].value(),
                                       mirror=self.args["mirror"].value(),
                                       options=self.args["options"].value(),
                                       with_comment=self.args["with_comment"].value())


@_pimpdoc()
class Ls(Call):
    """
    List source package
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield Codenames(["--codenames", "-C"], default=Value.ALL_CODENAMES)
        yield StrArgument(["--version", "-V"], default="", doc="Limit to exactly this version")
        yield StrArgument(["--min-version", "-M"], default="", doc="Limit to this version or greater")

    def run(self):
        for repository in self.args["repositories"].objects():
            ls = repository.mbd_reprepro.ls(self.args["source"].value(),
                                            codenames=self.args["codenames"].value(),
                                            version=self.args["version"].value(),
                                            min_version=self.args["min_version"].value())
            if ls:
                self.result[repository.identity] = ls


@_pimpdoc()
class Show(Ls):
    """
    Show source package
    """

    def run(self):
        super().run()
        for value in self.result.values():
            value.enrich()


MULT_VERSIONS_PER_DIST_NOTE = "Use as safeguard, or for rare cases of multiple version of the same package in one distribution (in different components)"


class Debdiff(Call):
    """
    Compare two internal source packages
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Repository(["repository"])
        yield StrArgument(["--versiona", "-a"], default="", doc="Version a to compare")
        yield StrArgument(["--versionb", "-b"], default="", doc="Version b to compare")

    def run(self):
        repository = self.args["repository"].object()
        repositories_path = mini_buildd.config.ROUTES["repositories"].path
        dsca = repository.mbd_dsc_pool_path(self.args["source"].value(), self.args["versiona"].value())
        dscb = repository.mbd_dsc_pool_path(self.args["source"].value(), self.args["versionb"].value())
        self.result = mini_buildd.call.Call(["debdiff", repositories_path.join(dsca), repositories_path.join(dscb)]).stdout


@_pimpdoc()
class Migrate(_Staff, _Confirm, Call):
    """
    Migrate source package

    Migrates a source package along with all its binary packages. If
    run for a rollback distribution, this will perform a rollback
    restore.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["distribution"], choices=Value.MIGRATABLE_DISTRIBUTIONS)
        yield BoolArgument(["--full", "-F"], doc="migrate all 'migrates_to' suites up (f.e. unstable->testing->stable)")
        yield StrArgument(["--version", "-V"], default="", doc=f"Migrate exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")

    def run(self):
        repository, distribution, suite = mini_buildd.mdls().repository.parse_dist(self.args["distribution"].dist())
        version = self.args["version"].value()
        repository.mbd_package_migrate(self.args["source"].value(),
                                       distribution,
                                       suite,
                                       full=self.args["full"].value(),
                                       rollback=self.args["distribution"].dist().rollback_no,
                                       version=version if version else None)


@_pimpdoc()
class Remove(_Admin, _Confirm, Call):
    """
    Remove source package

    Removes a source package along with all its binary packages.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["distribution"])
        yield BoolArgument(["--without-rollback"], doc="don't copy to rollback distribution")
        yield StrArgument(["--version", "-V"], default="", doc=f"Remove exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")

    def run(self):
        repository, distribution, suite = mini_buildd.mdls().repository.parse_dist(self.args["distribution"].dist())
        version = self.args["version"].value()
        repository.mbd_package_remove(self.args["source"].value(),
                                      distribution,
                                      suite,
                                      rollback=self.args["distribution"].dist().rollback_no,
                                      version=version if version else None,
                                      without_rollback=self.args["without_rollback"].value())


PORT_RESULT_DOC = (
    "JSON Result (dict):",
    "  list of dicts of all items uploaded",
)


@_pimpdoc(PORT_RESULT_DOC)
class Port(_Staff, _Running, _Confirm, Call):
    """
    Port internal source package

    An internal 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of the given locally-installed
    package.

    When from-distribution equals to_distribution, a rebuild will be done.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["from_distribution"])
        yield Distributions(["to_distributions"], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield StrArgument(["--version", "-V"], default="", doc=f"Port exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")
        yield UploadOptions(["--options", "-O"])

    def run(self):
        self.result = []
        version = self.args["version"].value()
        for to_diststr in self.args["to_distributions"].value():
            self.result.append(mini_buildd.package.port(self.args["source"].value(),
                                                        self.args["from_distribution"].value(),
                                                        to_diststr,
                                                        version=version if version else None,
                                                        options=self.args["options"].value().split("|")))


@_pimpdoc(PORT_RESULT_DOC)
class PortExt(_Staff, _Running, _Confirm, Call):
    """
    Port external source package

    An external 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of any given source package.
    """

    @classmethod
    def iarguments(cls):
        yield UrlArgument(["dsc"], doc="URL of any Debian source package (dsc) to port")
        yield BoolArgument(["--allow-unauthenticated", "-u"], doc="Don't verify downloaded DSC against Debian keyrings (see ``man dget``)")
        yield Distributions(["distributions"], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield UploadOptions(["--options", "-O"])

    def run(self):
        self.result = []
        for diststr in self.args["distributions"].value():
            self.result.append(mini_buildd.package.port_ext(self.args["dsc"].value(),
                                                            diststr,
                                                            options=self.args["options"].value().split("|"),
                                                            allow_unauthenticated=self.args["allow_unauthenticated"].value()))


@_pimpdoc()
class Retry(_Staff, _Running, _Confirm, Call):
    """
    Retry a previously failed source package

    JSON Result:

    * changes: Changes file that has been re-uploaded.
    """

    BKEY_FORMAT = "<source>/<version>/<timecode>/source[ <arch>]"
    BKEY_REGEX = re.compile(r"[^/]+/[^/]+/[^/]+/[^/]+")

    @classmethod
    def iarguments(cls):
        yield StrArgument(["bkey"], choices=Value.LAST_FAILED_BKEYS, doc=f"Package bkey ('{cls.BKEY_FORMAT}') of a past packaging try; 'extra.bkey' of a FAILED packaging event")

    def run(self):
        bkey = self.args["bkey"].value()
        if not self.BKEY_REGEX.match(bkey):
            raise mini_buildd.HTTPBadRequest(f"Wrong bkey format. Should be '{self.BKEY_FORMAT}'")

        events_path = mini_buildd.config.ROUTES["events"].path.new_sub([bkey])
        if not os.path.exists(events_path.full):
            raise mini_buildd.HTTPBadRequest(f"No such event path: {bkey}")

        # Check that there is exactly one failed event in the given path
        failed_events = glob.glob(events_path.join("*_FAILED.json"))
        if len(failed_events) != 1:
            raise mini_buildd.HTTPBadRequest(f"{len(failed_events)} failed events found (we need exactly 1). Check your bkey")
        failed_event = mini_buildd.events.Event.load(failed_events[0])
        LOG.debug(f"Retry: Found valid FAILED event: {failed_event}")

        # Find all changes in that path
        changes_files = glob.glob(events_path.join("*.changes"))
        if len(changes_files) != 1:
            raise mini_buildd.HTTPBadRequest(f"{len(changes_files)} changes found (we need exactly 1). Check your bkey")
        changes = changes_files[0]
        LOG.debug(f"Retry: Found solitaire changes file: {changes}")

        # Should be save now to re-upload the changes files found in path
        mini_buildd.changes.Base(changes).upload(mini_buildd.get_daemon().model.mbd_get_ftp_endpoint(), force=True)
        self.result = os.path.basename(changes)


@_pimpdoc()
class Cancel(_Staff, _Running, _Confirm, Call):
    """Cancel an ongoing package build"""

    @classmethod
    def iarguments(cls):
        yield StrArgument(["bkey"], choices=Value.CURRENT_BUILDS, doc="build key ('<source>/<version>/<timecode>/<arch>' of ongoing builds; 'extra.bkey' in events)")

    def run(self):
        mini_buildd.get_daemon().builder.cancel(self.args["bkey"].value(), f"{self.request.user}")
        self.result = self.args["bkey"].value()


@_pimpdoc()
class SetUserKey(_Login, _Confirm, Call):
    """
    Set a user's GnuPG public key
    """

    @classmethod
    def category(cls):
        return cls.CATEGORIES[1]

    @classmethod
    def iarguments(cls):
        yield MultilineStrArgument(["key"], doc="GnuPG public key; multiline inputs will be handled as ascii armored full key, one-liners as key ids")

    def run(self):
        uploader = self.request.user.uploader
        uploader.Admin.mbd_remove(uploader)
        key = self.args["key"].value()

        if "\n" in key:
            LOG.debug("Using given key argument as full ascii-armored GPG key")
            uploader.key_id = ""
            uploader.key = key
        else:
            LOG.debug("Using given key argument as key ID")
            uploader.key_id = key
            uploader.key = ""

        uploader.Admin.mbd_prepare(uploader)
        uploader.Admin.mbd_check(uploader)
        LOG.warning(f"Uploader profile changed: {uploader} (must be (re-)activated by mini-buildd staff before you can actually use it)")


@_pimpdoc()
class Subscribe(_Login, Call):
    """
    Subscribe to (email) notifications
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"], default="", extra_doc="\n\nLeave empty for all source packages")
        yield Distribution(["distribution"], default="", choices=Value.PREPARED_DISTRIBUTIONS, extra_doc="\n\nLeave empty for all distributions")

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.source = self.args["source"].value()
        self.sourcestr = "" if self.source is None else self.source
        self.distribution = self.args["distribution"].value()
        self.distributionstr = "" if self.distribution is None else self.distribution

    def run(self):
        s, created = mini_buildd.mdls().subscription.Subscription.objects.get_or_create(
            subscriber=self.request.user,
            package=self.sourcestr,
            distribution=self.distributionstr,
        )
        self.result = {"created": created, "source": s.package, "distribution": s.distribution}


@_pimpdoc()
class Unsubscribe(Subscribe):
    """
    Unsubscribe from (email) notifications
    """

    def run(self):
        self.result = []
        for s in mini_buildd.mdls().subscription.Subscription.objects.filter(subscriber=self.request.user):
            if (self.sourcestr == s.package) and (self.distributionstr == s.distribution):
                s.delete()
                self.result.append({"source": s.package, "distribution": s.distribution})


@_pimpdoc()
class RemakeChroots(_Admin, _Confirm, Call):
    """
    Remake chroots

    Run actions 'remove', 'prepare', 'check' and 'activate'.

    Daemon will be stopped while running.
    """

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--keys"], default=Value.ALL_CHROOTS, choices=Value.ALL_CHROOTS, doc="chroot keys (<codename>:<arch>)")

    def run(self):
        with mini_buildd.daemon.Stopped():
            chroots = mini_buildd.mdls().chroot.Chroot.objects.none()
            for key in self.args["keys"].value():
                codename, arch = key.split(":")
                chroots |= mini_buildd.mdls().chroot.Chroot.objects.filter(source__codename=codename, architecture__name=arch)
            mini_buildd.mdls().chroot.Chroot.Admin.mbd_actions(self.request, chroots, ["remove", "prepare", "check", "activate"])


@_pimpdoc()
class Power(_Admin, _Confirm, Call):
    """
    Power Daemon on or off (toggles by default)

    This essentially stops accepting incoming, and forcibly stops any possibly running builds.

    This state is *not persisted*. Please *deactivate* the Daemon instance via :mbdpage:`setup` to make the state persist over *mini-buildd service* restarts.
    """

    @classmethod
    def iarguments(cls):
        yield ChoiceArgument(["to_state"], choices=[o.name for o in OnOff] + [Argument.SERVER_DEFAULT], default=Value.POWER_TOGGLE, doc="power state to set to")

    def run(self):
        self.result["state_pre"] = OnOff(mini_buildd.get_daemon().is_alive()).name
        {OnOff.ON.name: mini_buildd.get_daemon().mbd_start, OnOff.OFF.name: mini_buildd.get_daemon().mbd_stop}[self.args["to_state"].value()]()
        self.result["state"] = OnOff(mini_buildd.get_daemon().is_alive()).name


@_pimpdoc()
class Wake(_Admin, _Confirm, Call):
    """
    Wake a remote instance
    """

    @classmethod
    def iarguments(cls):
        yield StrArgument(["--remote", "-r"], choices=Value.ALL_REMOTES)
        yield IntArgument(["--sleep", "-s"], default=5, doc="Sleep between wake attempts")
        yield IntArgument(["--attempts", "-a"], default=3, doc="Max number attempts")

    def run(self):
        remote = mini_buildd.mdls().gnupg.Remote.objects.get(http=self.args["remote"].value())
        self.result = remote.mbd_get_status(wake=True, wake_sleep=self.args["sleep"].value(), wake_attempts=self.args["attempts"].value())


@_pimpdoc()
class Handshake(Call):
    """
    Check if signed message matches a remote, reply our signed message on success

    This is for internal use only.
    """

    @classmethod
    def category(cls):
        return cls.CATEGORIES[2]

    @classmethod
    def iarguments(cls):
        yield MultilineStrArgument(["--signed-message", "-S"])

    def run(self):
        signed_message = self.args["signed_message"].value()
        for r in mini_buildd.mdls().gnupg.Remote.objects.all():
            try:
                r.mbd_verify(signed_message)
                self.result = mini_buildd.get_daemon().handshake_message()
                LOG.debug(f"Remote handshake ok: '{r}': {r.key_long_id}: {r.key_name}")
                return
            except Exception as e:
                mini_buildd.log_exception(LOG, f"Remote handshake failed for '{r}'", e)
        raise mini_buildd.HTTPBadRequest(f"GnuPG handshake failed: No remote for public key on {mini_buildd.http_endpoint()}")


@_pimpdoc()
class Cronjob(_Admin, _Confirm, Call):
    """
    Run a cron job now (out of schedule)
    """

    @classmethod
    def iarguments(cls):
        yield StrArgument(["id"], choices=Value.ALL_CRONJOBS)

    def run(self):
        job = mini_buildd.get_daemon().crontab.get(self.args["id"].value())
        self.result[job.id()] = job.run()


@_pimpdoc()
class Uploaders(_Admin, _Running, Call):
    """
    Get upload permissions for repositories
    """

    @classmethod
    def iarguments(cls):
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)

    def run(self):
        for r in self.args["repositories"].objects():
            self.result[r.identity] = {"allow_unauthenticated_uploads": r.allow_unauthenticated_uploads}
            with closing(mini_buildd.daemon.UploadersKeyring(r.identity)) as gpg:
                self.result[r.identity]["uploaders"] = gpg.get_pub_keys_infos()


@_pimpdoc()
class SnapshotLs(_Running, Call):
    """
    Get list of repository snapshots for a distribution

    JSON Result (dict):
      dict: <distribution>: [snasphot,...]: List of snapshots for the given distribution.
    """

    @classmethod
    def iarguments(cls):
        yield Distribution(["distribution"])

    def run(self):
        diststr = self.args["distribution"].value()
        self.result = diststr2repository(diststr).mbd_reprepro.get_snapshots(diststr)


@_pimpdoc()
class SnapshotCreate(_Admin, _Confirm, SnapshotLs):
    """
    Create a repository snapshot
    """

    @classmethod
    def iarguments(cls):
        yield from SnapshotLs.iarguments()
        yield StrArgument(["name"], doc="snapshot name")

    def run(self):
        diststr = self.args["distribution"].value()
        diststr2repository(diststr).mbd_reprepro.gen_snapshot(diststr, self.args["name"].value())


class SnapshotDelete(SnapshotCreate):
    """
    Delete a repository snapshot
    """

    def run(self):
        diststr = self.args["distribution"].value()
        diststr2repository(diststr).mbd_reprepro.del_snapshot(diststr, self.args["name"].value())


@_pimpdoc()
class Debmirror(_Admin, _Confirm, Call):
    """
    Make local partial repository mirror via :debpkg:`debmirror`

    This may be useful if you plan on publishing a stripped-down (f.e.,
    only certain repos, only ``stable``, omit rollbacks) variant of your
    repo somewhere remote.

    .. error:: debmirror (:debbug:`819925`): ``apt update`` fails on ``experimental`` suites (contents not mirrored)

      This happens only on systems where APT is configured to download contents, most likely just because
      ``apt-file`` is installed. So, the easiest workaround is::

        apt purge apt-file

      Closest to an actual fix is to install debmirror variant '+abfixes' from
      ``Hellfield Archive``, where the "bug" has been fixed (Dec 2022: at least
      available for bullseye).
    """

    PROGRAM = "/usr/bin/debmirror"
    DEB = "debmirror"

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--suites", "-S"], default=Value.ALL_SUITES, choices=Value.ALL_SUITES, doc="Suite names (like 'stable', 'unstable')")
        yield Rollbacks(["--rollbacks", "-r"])
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield StrArgument(["--architectures", "-A"],
                          default="amd64,i386,arm64,armhf,armel,s390x,ppc64el,mipsel,mips64el",
                          doc=("Architectures to mirror (``--arch`` in debmirror)\n\n"
                               "Usually, you just want all, so the default already lists all currently supported Debian/Ubuntu architectures (but will only mirror the architectures actually found)"))
        yield StrArgument(["--components", "-C"],
                          default="main,contrib,non-free,main/debian-installer,multiverse,restricted,universe",
                          doc=("Components to mirror (``--section`` in debmirror)\n\n"
                               "Usually, you just want all, so the default already lists all known Debian/Ubuntu components (but will only mirror the components actually found)"))
        yield StrArgument(["--destination", "-D"],
                          default=Value.DEFAULT_DEBMIRROR_DESTINATION,
                          doc=("Mirror destination dir (``mirrordir`` in debmirror)\n"
                               "\n"
                               "* ``{}`` will be replaced by repository identity\n"
                               "* Directory must be accessible by user ``mini-buildd``\n"
                               "\n"
                               "*BE CAREFUL* with this value -- anything in this directory will be *HAPPILY REPLACED* by the mirror only\n"))

    def run(self):
        mini_buildd.check_program(self.PROGRAM, self.DEB)
        for r in self.args["repositories"].objects():
            debmirror = [
                self.PROGRAM,
                "--verbose",
                "--ignore-release-gpg",  # change
                "--diff", "none",
                "--getcontents",
                "--rsync-extra", "none",
                "--host", mini_buildd.http_endpoint().hopo(),
                "--method", mini_buildd.http_endpoint().scheme(),
                "--root", f"/repositories/{r.identity}",
                "--dist", ",".join(r.mbd_get_diststrs(frollbacks=self.args["rollbacks"].range, suite__name__in=self.args["suites"].value())),
                "--arch", self.args["architectures"].value(),
                "--section", self.args["components"].value(),
                self.args["destination"].value().format(r.identity),
            ]
            LOG.debug(debmirror)
            mini_buildd.call.Call(debmirror).check()


@_pimpdoc(PORT_RESULT_DOC)
class KeyringPackages(_Admin, _Running, _Confirm, Call):
    """
    Build keyring packages

    .. tip:: **keyring-packages**: No compat for urold (``apt-key add``)

      Since ``2.x``, keyring packages will use ``/etc/apt/trusted.gpg.d/<foo>.gpg``, not deprecated ``apt-key add <foo>``.

      In Debian, this is supported since ``wheezy (2013)``.

      For distributions ``<= squeeze`` (apt versions ``~<= 0.8.x``), you would manually have to run ``apt-key add /etc/apt/trusted.gpg.d/<foo>.gpg`` after installation of the keyring package.
    """

    @classmethod
    def iarguments(cls):
        yield Distributions(["--distributions", "-D"], default=Value.ACTIVE_KEYRING_DISTRIBUTIONS)
        yield BoolArgument(["--without-migration", "-M"], doc="migrate packages")

    def run(self):
        self.result = []
        events = mini_buildd.events.Attach(mini_buildd.get_daemon().events)

        for diststr in self.args["distributions"].value():
            repository, distribution, suite = mini_buildd.mdls().repository.parse_diststr(diststr)
            if not suite.build_keyring_package:
                raise mini_buildd.HTTPBadRequest(f"Keyring package to non-keyring suite requested (see 'build_keyring_package' flag): '{diststr}'")
            self.result.append(mini_buildd.package.upload_template_package(mini_buildd.package.KeyringPackage(), diststr))

        def unfinished():
            return [upload for upload in self.result if "event" not in upload]

        while unfinished():
            event = events.get()
            for upload in unfinished():
                if event.match(types=[mini_buildd.events.Type.INSTALLED, mini_buildd.events.Type.FAILED, mini_buildd.events.Type.REJECTED], distribution=upload["distribution"], source=upload["source"], version=upload["version"]):
                    LOG.debug(f"Keyring package result: {event}")
                    upload["event"] = event.type.name
                    if (event.type == mini_buildd.events.Type.INSTALLED) and not self.args["without_migration"].value():
                        repository, distribution, suite = mini_buildd.mdls().repository.parse_diststr(upload["distribution"])
                        repository.mbd_package_migrate(upload["source"], distribution, suite, full=True, version=upload["version"])


@_pimpdoc(PORT_RESULT_DOC)
class TestPackages(_Admin, _Running, _Confirm, Call):
    """
    Build test packages
    """

    __TEMPLATES = ["mbd-test-archall", "mbd-test-cpp", "mbd-test-ftbfs"]

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--sources", "-S"], default=cls.__TEMPLATES, choices=cls.__TEMPLATES, doc="test source packages to use")
        yield Distributions(["--distributions", "-D"], default=Value.ACTIVE_EXPERIMENTAL_DISTRIBUTIONS, choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield Distributions(["--auto-ports", "-A"], default=[], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield BoolArgument(["--with-check", "-c"], doc="Check for correct packager results")

    def run(self):
        self.result = []
        events = mini_buildd.events.Attach(mini_buildd.get_daemon().events)

        for source in self.args["sources"].value():
            for diststr in self.args["distributions"].value():
                self.result.append(mini_buildd.package.upload_template_package(mini_buildd.package.TestPackage(source, auto_ports=self.args["auto_ports"].value()), diststr))

        if self.args["with_check"].value():
            def unfinished():
                return [upload for upload in self.result if "event" not in upload]

            while unfinished():
                event = events.get()
                for upload in unfinished():
                    if event.match(types=[mini_buildd.events.Type.INSTALLED, mini_buildd.events.Type.FAILED, mini_buildd.events.Type.REJECTED], distribution=upload["distribution"], source=upload["source"], version=upload["version"]):
                        upload["event"] = event.type.name
                        if (event.type == mini_buildd.events.Type.FAILED and upload["source"] in ["mbd-test-archall", "mbd-test-cpp"]) or \
                           (event.type == mini_buildd.events.Type.INSTALLED and upload["source"] in ["mbd-test-ftbfs"]):
                            raise mini_buildd.HTTPBadRequest(f"Test package failed: {event}")
                        LOG.debug(f"Test package result: {event}")


@_pimpdoc()
class Setup(_Admin, _Confirm, Call):
    """
    Create, update or inspect your setup

    Daemon will be stopped while running, and started when finished (i.e., even if it was not running before).
    """

    @classmethod
    def iarguments(cls):
        yield BoolArgument(["--save"], doc="Save command line from this (successful) run as default for future runs", header="Run Options")
        yield StrArgument(["--update"], default="", doc="Update existing instances; 'all' to update all, empty string to never update, <hash_id>,.. (see a previous run) to update selected instances")
        yield StrArgument(["--pca"], default="", doc="Prepare, check and create instances; 'all' to pca all, empty string to never pca, <hash_id>,.. (see a previous run) to pca selected instances")

        yield StrArgument(["--identity"], default=Value.DEFAULT_IDENTITY, doc="instance identity (for keyring package names, dput config, ...)", header="Daemon")
        yield StrArgument(["--ftp-endpoint"], default=Value.DEFAULT_FTP_ENDPOINT, doc="ftp (incoming) network endpoint")

        yield ListArgument(["--archives", "-A"], default=[], doc="add arbitrary archives", header="Archives")
        yield ListArgument(["--archives-from-origin"], default=[], choices=SETUP["origin"].keys(), doc="add original archives from these origins")
        yield BoolArgument(["--archives-from-apt-sources"], doc="add archives guessed from your local sources.list")
        yield BoolArgument(["--archives-from-proxy"], doc="add archives guessed from a locally running apt-cacher-ng")

        yield ListArgument(["--sources"], default=[], choices=CODENAMES.keys(), doc="manually select codenames to setup sources for", header="Sources")
        yield ListArgument(["--sources-from-origin"], default=[], choices=SETUP["origin"].keys(), doc="add 'supported' (as per ``distro-info``) set of source codenames from these origins")
        yield ListArgument(["--sources-from-origin-with-lts"], default=[], choices=SETUP["origin"].keys(), doc="add 'supported' plus 'lts' (as per ``distro-info`` -- adds 'lts', 'elts' for Debian and 'esm' for Ubuntu) set of source codenames from these origins")
        yield ListArgument(["--sources-from-origin-with-all"], default=[], choices=SETUP["origin"].keys(), doc="add all source codenames with a working setup from these origins (i.e., includes very old ones)")

        yield ListArgument(["--repositories"], default=[], doc=f"repositories to setup \'<repo_id>[/({'|'.join(SETUP['repository'].keys())})]\'", header="Repositories")
        yield ListArgument(["--repositories-extra-architectures"], default=[],
                           doc="""add these extra architectures to distributions

We will always add all architectures your host natively supports.
""")

        yield ListArgument(["--chroots"], default=[], choices=CODENAMES.keys(), doc="codenames to setup chroots for", header="Chroots")
        yield BoolArgument(["--chroots-from-sources"], doc="add chroots for all base sources found")
        yield ListArgument(["--chroots-extra-architectures"], default=[], choices=mini_buildd.dist.Archs.available(),
                           doc=f"""try to create chroots for these extra architectures

We will always try to create chroots for architectures this host natively supports: {','.join(mini_buildd.dist.Archs.native())}.

.. tip:: With :debpkg:`qemu-user-static` installed, you also have seamless access to foreign archs (albeit with a speed penalty)

Chroots that cannot be build for a given architecture will just be skipped.
""")

        yield ChoiceArgument(["--chroots-backend", "-C"], default=Value.DEFAULT_CHROOT_BACKEND, choices=["Dir", "File", "LVM", "LoopLVM", "BtrfsSnapshot"], doc="chroot backend to use")

        yield ListArgument(["--remotes"], default=[], doc=f"remotes to add. {inspect.getdoc(mini_buildd.net.ClientEndpoint)}", header="Remotes")

    class Instance:
        def diff(self):
            diff = {
                "setup": {
                    "fields": [],
                    "diff": {},
                },
                "model": {
                    "fields": [],
                    "diff": {},
                },
            }

            setup = {**self.identifiers, **self.options}
            target = self.model(**setup)
            for field in (f for f in self.model.mbd_get_fields(exclude=["id", "status", "last_checked", "auto_reactivate", "pickled_data", "ping"]) if not f.is_relation):
                data = diff["setup"] if field.name in setup else diff["model"]

                data["fields"].append(field.name)
                current_value = None if self.obj is None else getattr(self.obj, field.name, None)
                target_value = getattr(target, field.name, None)
                if current_value != target_value:
                    data["diff"][field.name] = {
                        "current": str(current_value),
                        "target": str(target_value),
                    }
            return diff

        @classmethod
        def _add_m2m_diff(cls, diff, field, expected, current):
            diff["setup"]["fields"].append(field)
            _expected = set(expected)
            _current = set(current)
            if _expected - _current:
                diff["setup"]["diff"].setdefault(field, {})["missing"] = list(_expected - _current)
            if _current - _expected:
                diff["setup"]["diff"].setdefault(field, {})["unknown"] = list(_current - _expected)

        def __init__(self, call, model, options, update_args, **identifiers):
            self.call, self.model, self.options, self.update_args, self.identifiers = call, model, options, update_args, identifiers
            LOG.debug(f"Instance: {self.model}({self.identifiers})")

            self.identity = {
                "class": repr(self.model),
                "identifiers": {k: repr(v) for k, v in self.identifiers.items()},
            }
            self.identity_hash = hashlib.sha256(json.dumps(self.identity, sort_keys=True).encode(mini_buildd.config.CHAR_ENCODING)).hexdigest()

            self.obj = model.mbd_get_or_none(**self.identifiers)
            self.info = {
                "identity": self.identity,
                "identity_hash": self.identity_hash,
                "identity_str": f"{self.model.__name__}({self.identifiers})",
                "updated": False,
                "created": False,
                "pca": False,
                "diff": None,
                "status": None,
            }

            if self.call.args["update"].value() in ["all"] + self.identity_hash.split(","):
                if self.obj is None:
                    self.obj = self.model(**self.identifiers, **self.options)
                    self.obj.save()
                    self.info["created"] = True
                else:
                    self.info["updated"] = self.model.objects.filter(**self.identifiers).update(**self.options) > 0
                    self.obj = self.model.mbd_get_or_none(**self.identifiers)  # Get updated instance

                if self.obj is not None:
                    self.update()
                    self.obj.save()

            if self.call.args["pca"].value() in ["all"] + self.identity_hash.split(","):
                if self.obj is not None and issubclass(self.model, mini_buildd.mdls().base.StatusModel):
                    self.model.Admin.mbd_pca(self.call.request, [self.obj])
                    self.info["pca"] = True

            self.info["diff"] = self.diff()
            self.info["status"] = "None" if self.obj is None else self.obj.get_status_display() if issubclass(self.model, mini_buildd.mdls().base.StatusModel) else "Exists"
            if self.obj is not None and not self.model.__name__.endswith("Option"):
                # Help for HTML template only
                self.info["change_instance_url"] = f"admin:mini_buildd_{self.model.__name__.lower()}_change"
                self.info["change_instance_pk"] = str(self.obj.pk)
            self.call.result.append(self.info)

        def update(self):
            pass

    @classmethod
    def ilocal_archive_urls(cls):
        try:
            import aptsources.sourceslist
            for src in (src for src in aptsources.sourceslist.SourcesList() if not src.invalid and not src.disabled):
                # These URLs come from the user. 'normalize' the uri first to have exactly one trailing slash.
                yield src.uri.rstrip("/") + "/"
        except BaseException as e:
            mini_buildd.log_exception(LOG, "Failed to scan local sources.lists for default mirrors ('python-apt' not installed?)", e)

    @classmethod
    def iapt_cacher_archive_urls(cls):
        url = mini_buildd.net.detect_apt_cacher_ng(url=f"http://{mini_buildd.config.HOSTNAME_FQDN}:3142")
        if url:
            LOG.debug(f"Local apt-cacher-ng detected: {url}")
            for setup in SETUP["origin"].values():
                for path in setup.get("archive_paths", []):
                    LOG.debug(f"Local proxy archive: '{url}/{path}/'")
                    yield f"{url}/{path}/"

    def _check_codenames(self, codenames):
        for codename in codenames:
            if CODENAMES.get(codename) is None:
                raise mini_buildd.HTTPBadRequest(f"No setup for codename '{codename}'")

    def _setup(self):
        # Daemon
        Setup.Instance(self,
                       mini_buildd.mdls().daemon.Daemon,
                       {
                           "identity": self.args["identity"].value(),
                           "ftpd_bind": self.args["ftp_endpoint"].value(),
                       },
                       {})

        # Archives
        archive_urls = set(self.args["archives"].value())
        if self.args["archives_from_apt_sources"].value():
            archive_urls.update(self.ilocal_archive_urls())
        if self.args["archives_from_proxy"].value():
            archive_urls.update(self.iapt_cacher_archive_urls())
        for v in self.args["archives_from_origin"].value():
            archive_urls.update(SETUP["origin"][v]["archive"])

        for url in archive_urls:
            Setup.Instance(self, mini_buildd.mdls().source.Archive, {}, {}, url=url)

        # Sources
        sources_codenames = set(self.args["sources"].value())
        for v in self.args["sources_from_origin"].value():
            sources_codenames.update(CODENAMES.get_supported(v))
        for v in self.args["sources_from_origin_with_lts"].value():
            sources_codenames.update(CODENAMES.get_supported_plus_lts(v))
        for v in self.args["sources_from_origin_with_all"].value():
            sources_codenames.update(CODENAMES.get_working(v))
        self._check_codenames(sources_codenames)

        class SourceInstance(Setup.Instance):
            def update(self):
                # apt_keys (m2m) need to be updated manually, with save() (x_or_create can't be used)
                self.obj.apt_keys.clear()
                for long_key_id in self.update_args["apt_keys"]:
                    matching_keys = mini_buildd.mdls().gnupg.AptKey.mbd_filter_key(long_key_id)
                    if matching_keys:
                        self.obj.apt_keys.add(matching_keys[0])
                    else:
                        self.obj.apt_keys.add(Setup.Instance(self.call, mini_buildd.mdls().gnupg.AptKey, {}, {}, key_id=long_key_id).obj)

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "apt_keys", self.update_args["apt_keys"], [a.key_long_id for a in self.obj.apt_keys.all()])
                return diff

        for codename in sources_codenames:
            for c, s in CODENAMES[codename]["sources"].items():
                origin = s.get("origin", CODENAMES[codename]["origin"])
                source = SourceInstance(self, mini_buildd.mdls().source.Source, s.get("options", {}), {"apt_keys": s.get("apt_keys", CODENAMES[codename].get("apt_keys"))}, origin=origin, codename=c)
                # Smartly add prio source: Security gets prio=500, others prio=1 (opt-in).
                Setup.Instance(self, mini_buildd.mdls().source.PrioritySource, {}, {}, source=source.obj, priority=500 if CODENAMES.is_security(origin, c) else 1)

        # Layouts
        for layout_name, layout_setup in SETUP["layout"].items():
            instance = Setup.Instance(self, mini_buildd.mdls().distribution.Layout, layout_setup.get("options", {}), {}, name=layout_name)
            for suite_name, suite_setup in layout_setup["suites"].items():
                suite_instance = Setup.Instance(self, mini_buildd.mdls().distribution.Suite, {}, {}, name=suite_name)
                Setup.Instance(self,
                               mini_buildd.mdls().distribution.SuiteOption,
                               {
                                   **suite_setup.get("options", {}),
                                   "extra_options": f"Rollback: {layout_setup['rollback'].get(suite_name, 0)}\n",
                                   "migrates_to": mini_buildd.mdls().distribution.SuiteOption.mbd_get_or_none(layout=instance.obj, suite=mini_buildd.mdls().distribution.Suite.mbd_get_or_none(name=suite_setup.get("migrates_to"))),
                               },
                               {},
                               layout=instance.obj,
                               suite=suite_instance.obj)

        # Distributions
        class DistributionInstance(Setup.Instance):
            def update(self):
                # Auto-add known default components or Origin
                self.obj.components.set(mini_buildd.mdls().source.Component.objects.filter(name__in=SETUP["origin"][self.obj.base_source.origin]["default_components"]))

                # Auto-add extra (prio) sources that start with our base_source's codename
                self.obj.extra_sources.set(mini_buildd.mdls().source.PrioritySource.objects.filter(source__codename__regex=fr"^{self.obj.base_source.codename}[-/]"))

                # Auto-add architectures to build for
                archall_set = False  # Use first non-optional arch to build arch "all"
                for architecture in self.obj.base_source.architectures.all():
                    if architecture.name in (mini_buildd.dist.Archs.native() + self.call.args["repositories_extra_architectures"].value()):
                        optional = architecture.name in self.update_args["setup"].get("distribution", {}).get("arch_optional", [])
                        instance = Setup.Instance(
                            self.call,
                            mini_buildd.mdls().distribution.ArchitectureOption,
                            {
                                "build_architecture_all": not archall_set and not optional,
                                "optional": optional,
                            },
                            {},
                            architecture=architecture,
                            distribution=self.obj)
                        if instance.obj.build_architecture_all:
                            archall_set = True

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "components",
                                       SETUP["origin"][self.obj.base_source.origin]["default_components"],
                                       [c.name for c in self.obj.components.all()])
                    self._add_m2m_diff(diff, "extra_sources",
                                       [s.source.codename for s in mini_buildd.mdls().source.PrioritySource.objects.filter(source__codename__regex=fr"^{self.obj.base_source.codename}[-/]")],
                                       [s.source.codename for s in self.obj.extra_sources.all()])
                return diff

        for codename in sources_codenames:
            DistributionInstance(self,
                                 mini_buildd.mdls().distribution.Distribution,
                                 CODENAMES[codename].get("distribution", {}).get("options", {}),
                                 {"setup": CODENAMES[codename]},
                                 base_source=mini_buildd.mdls().source.Source.mbd_get_or_none(codename=codename))

        # Repositories
        class RepositoryInstance(Setup.Instance):
            def update(self):
                for d in mini_buildd.mdls().distribution.Distribution.objects.filter(**self.update_args["setup"]["distribution_filter"]):
                    self.obj.distributions.add(d)

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "distributions",
                                       [d.base_source.codename for d in mini_buildd.mdls().distribution.Distribution.objects.filter(**self.update_args["setup"]["distribution_filter"])],
                                       [d.base_source.codename for d in self.obj.distributions.all()])
                return diff

        for repo_preset in self.args["repositories"].value():
            repo, dummy, preset = repo_preset.partition("/")
            preset = preset if preset else "default"
            repository_setup = SETUP["repository"][preset]
            options = repository_setup.get("options", {})
            options["layout"] = mini_buildd.mdls().distribution.Layout.mbd_get_or_none(name__exact=repository_setup["layout"])
            RepositoryInstance(self, mini_buildd.mdls().repository.Repository, options, {"setup": repository_setup}, identity=repo)

        # Chroots
        chroots_codenames = set(self.args["chroots"].value())
        if self.args["chroots_from_sources"].value():
            chroots_codenames.update(sources_codenames)
        self._check_codenames(chroots_codenames)

        cb_class = getattr(mini_buildd.mdls().chroot, f"{self.args['chroots_backend'].value()}Chroot")
        for codename in chroots_codenames:
            source = mini_buildd.mdls().source.Source.mbd_get_or_none(codename=codename)
            for arch in mini_buildd.dist.Archs.native() + self.args["chroots_extra_architectures"].value():
                architecture = Setup.Instance(self, mini_buildd.mdls().source.Architecture, {}, {}, name=arch)
                Setup.Instance(self, cb_class, CODENAMES[codename].get("chroot", {}).get("options", {}), {}, source=source, architecture=architecture.obj)

        # Remotes
        for ep in self.args["remotes"].value():
            Setup.Instance(self, mini_buildd.mdls().gnupg.Remote, {}, {}, http=ep)

    def run(self):
        self.result = []
        if self._LOCK.acquire(blocking=False):  # pylint: disable=consider-using-with
            try:
                with mini_buildd.daemon.Stopped(force_start=True):
                    self._setup()
                    if self.args["save"].value():
                        mini_buildd.get_daemon().model.mbd_setup_save(self.command_line(exclude=["update", "pca", "save"]))
            finally:
                self._LOCK.release()
        else:
            raise mini_buildd.HTTPUnavailable("API call 'setup' already running")

    @classmethod
    def _preset(cls, **kwargs):
        return Setup(update="all", chroots_from_sources=True, **kwargs)

    @classmethod
    def preset_debian(cls, **kwargs):
        return cls._preset(archives_from_origin="Debian", sources_from_origin="Debian", **kwargs)

    @classmethod
    def preset_ubuntu(cls, **kwargs):
        return cls._preset(archives_from_origin="Ubuntu", sources_from_origin="Ubuntu", **kwargs)

    @classmethod
    def preset_bootstrap(cls, **kwargs):
        return cls.preset_debian(repositories="test/test", **kwargs)

    @classmethod
    def preset_saved_or_bootstrap(cls):
        try:
            cl = mini_buildd.get_daemon().model.mbd_setup_get()
            if not cl:
                raise Exception("No saved setup")
            return {"desc": "Update with saved options", "setup": Setup.from_command_line(cl)}
        except BaseException as e:
            mini_buildd.log_exception(LOG, "Can't load saved setup (using empty setup)", e, level=logging.INFO)
            return {"desc": "Bootstrap (adds sandbox repo 'test' by default)", "setup": Setup.preset_bootstrap()}


class Calls(collections.OrderedDict):
    """Automatically collect all calls defined in this module, and make them accessible"""

    def __init__(self):
        super().__init__({c.name(): c for c in sys.modules[__name__].__dict__.values() if inspect.isclass(c) and issubclass(c, Call) and c != Call})


CALLS = Calls()
