import collections
import contextlib
import dataclasses
import datetime
import functools
import gzip
import hashlib
import itertools
import operator
import os
import re
import shutil
import subprocess
import textwrap
from contextlib import ExitStack
from tempfile import mkstemp
from typing import (
    Iterable,
    List,
    Optional,
    Set,
    Dict,
    Sequence,
    Tuple,
    Iterator,
    Callable,
    Literal,
    IO,
)

from debian.changelog import Changelog
from debian.substvars import Substvars

from debputy._deb_options_profiles import DebOptionsAndProfiles
from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable
from debputy.debhelper_emulation import (
    dhe_install_pkg_file_as_ctrl_file_if_present,
    dhe_dbgsym_root_dir,
)
from debputy.elf_util import find_all_elf_files, ELF_MAGIC
from debputy.filesystem_scan import FSPath, build_fs_from_root_dir, FSROOverlay
from debputy.highlevel_manifest import (
    HighLevelManifest,
    PackageTransformationDefinition,
    BinaryPackageData,
)
from debputy.maintscript_snippet import (
    ALL_CONTROL_SCRIPTS,
    MaintscriptSnippetContainer,
    STD_CONTROL_SCRIPTS,
)
from debputy.packages import BinaryPackage
from debputy.packaging.makeshlibs import compute_shlibs
from debputy.plugin.api import DpkgTriggerType
from debputy.plugin.api.impl import BinaryCtrlAccessorProvider
from debputy.plugin.api.impl_types import (
    MetadataDetectorPackageContextProvider,
    PluginProvidedTrigger,
    MetadataOrMaintscriptDetector,
)

from debputy.plugin.api.spec import FlushableSubstvars, VirtualPath
from debputy.util import (
    _error,
    ensure_dir,
    assume_not_none,
    perl_module_dirs,
    perlxs_api_dependency,
    detect_fakeroot,
    grouper,
    _info,
    xargs,
    escape_shell,
    generated_content_dir,
    print_command,
)


def generate_md5sums_file(control_output_dir: str, fs_root: FSPath) -> None:
    conffiles = os.path.join(control_output_dir, "conffiles")
    md5sums = os.path.join(control_output_dir, "md5sums")
    exclude = set()
    if os.path.isfile(conffiles):
        with open(conffiles, "rt") as fd:
            for line in fd:
                if not line.startswith("/"):
                    continue
                exclude.add("." + line.rstrip("\n"))
    had_content = False
    files = (
        path
        for path in fs_root.all_paths()
        if path.is_file and path.path not in exclude
    )
    with open(md5sums, "wt") as md5fd:
        for member in files:
            path = member.path
            assert path.startswith("./")
            path = path[2:]
            with member.open(byte_io=True) as f:
                file_hash = hashlib.md5()
                while chunk := f.read(8192):
                    file_hash.update(chunk)
            had_content = True
            md5fd.write(f"{file_hash.hexdigest()}  {path}\n")
    if not had_content:
        os.unlink(md5sums)


def install_or_generate_conffiles(
    binary_package: BinaryPackage, root_dir: str, fs_root: FSPath
) -> None:
    conffiles_dest = os.path.join(root_dir, "conffiles")
    debian_dir = FSROOverlay("debian", "debian", None)
    dhe_install_pkg_file_as_ctrl_file_if_present(
        debian_dir,
        binary_package,
        "conffiles",
        root_dir,
        0o0644,
    )
    etc_dir = fs_root.lookup("etc")
    if etc_dir:
        _add_conffiles(conffiles_dest, (p for p in etc_dir.all_paths() if p.is_file))
    if os.path.isfile(conffiles_dest):
        os.chmod(conffiles_dest, 0o0644)


def _filter_compress_paths(
    dctrl: BinaryPackage,
    target_doc_package: BinaryPackage,
) -> Callable[[FSPath], Iterator[FSPath]]:
    ignore_paths = {
        f"./usr/share/doc/{dctrl.name}/examples",
        f"./usr/share/doc/{target_doc_package.name}/examples",
    }
    ignore_dir_basenames = {
        "_sources",
    }
    ignore_basenames = {
        ".htaccess",
        "index.sgml",
        "objects.inv",
        "search_index.json",
        "copyright",
    }
    ignore_extensions = {
        ".xhtml",
        ".gif",
        ".png",
        ".jpg",
        ".jpeg",
        ".gz",
        ".taz",
        ".tgz",
        ".z",
        ".bz2",
        ".epub",
        ".jar",
        ".zip",
        ".odg",
        ".odp",
        ".odt",
        ".css",
        ".xz",
        ".lz",
        ".lzma",
        ".haddock",
        ".hs",
        ".woff",
        ".woff2",
        ".svg",
        ".svgz",
        ".js",
        ".devhelp2",
        ".map",  # Technically, dh_compress has this one case-sensitive
    }
    ignore_special_cases = ("-gz", "-z", "_z")

    def _filtered_walk(path: FSPath) -> Iterator[FSPath]:
        for path, children in path.walk():
            if path.path in ignore_paths or path.name in ignore_dir_basenames:
                children.clear()
                continue
            name = path.name
            if (
                path.is_symlink
                or not path.is_file
                or name in ignore_basenames
                or not path.has_fs_path
            ):
                continue

            name_lc = name.lower()
            _, ext = os.path.splitext(name_lc)

            if ext in ignore_extensions or name_lc.endswith(ignore_special_cases):
                continue
            yield path

    return _filtered_walk


def find_compressable_paths(
    dctrl: BinaryPackage,
    target_doc_package: BinaryPackage,
    fs_root: FSPath,
) -> Iterator[FSPath]:
    path_filter = _filter_compress_paths(dctrl, target_doc_package)

    for p, compress_size_threshold in (
        ("./usr/share/info", 0),
        ("./usr/share/man", 0),
        ("./usr/share/doc", 4096),
    ):
        path = fs_root.lookup(p)
        if path is None:
            continue
        paths = path_filter(path)
        if compress_size_threshold:
            # The special-case for changelog and NEWS is from dh_compress. Generally these files
            # have always been compressed regardless of their size.
            paths = (
                p
                for p in paths
                if p.stat().st_size > compress_size_threshold
                or p.name.startswith(("changelog", "NEWS"))
            )
        yield from paths
    x11_path = fs_root.lookup("./usr/share/fonts/X11")
    if x11_path:
        yield from (
            p for p in x11_path.all_paths() if p.is_file and p.name.endswith(".pcf")
        )


PERL_DEP_PROGRAM = 1
PERL_DEP_INDEP_PM_MODULE = 2
PERL_DEP_XS_MODULE = 4
PERL_DEP_ARCH_PM_MODULE = 8
PERL_DEP_MA_ANY_INCOMPATIBLE_TYPES = ~(PERL_DEP_PROGRAM | PERL_DEP_INDEP_PM_MODULE)
DESIRED_PERL_SHBANG_CMD = "/usr/bin/perl"
PERL_SHBANG_RE = re.compile(
    rb"""
    ^[#][!]\s*
    (/(?:\S+/perl|(?:usr/)?bin/env\s+perl))\s
""",
    re.VERBOSE | re.ASCII,
)


@functools.lru_cache(2)  # In practice, param will be "perl" or "perl-base"
def _dpkg_perl_version(package: str) -> str:
    dpkg_version = None
    lines = (
        subprocess.check_output(["dpkg", "-s", package])
        .decode("utf-8")
        .splitlines(keepends=False)
    )
    for line in lines:
        if line.startswith("Version: "):
            dpkg_version = line[8:].strip()
            break
    assert dpkg_version is not None
    return dpkg_version


def handle_perl_code(
    dctrl_bin: BinaryPackage,
    dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
    fs_root: FSPath,
    substvars: Substvars,
) -> None:
    known_perl_inc_dirs = perl_module_dirs(dpkg_architecture_variables, dctrl_bin)
    detected_dep_requirements = 0

    # MakeMaker always makes lib and share dirs, but typically only one directory is actually used.
    for perl_inc_dir in known_perl_inc_dirs:
        p = fs_root.lookup(perl_inc_dir)
        if p and p.is_dir:
            p.prune_if_empty_dir()

    # FIXME: 80% of this belongs in a metadata detector, but that requires us to expose .walk() in the public API,
    #  which will not be today.
    for d, pm_mode in [
        (known_perl_inc_dirs.vendorlib, PERL_DEP_INDEP_PM_MODULE),
        (known_perl_inc_dirs.vendorarch, PERL_DEP_ARCH_PM_MODULE),
    ]:
        inc_dir = fs_root.lookup(d)
        if not inc_dir:
            continue
        for path in inc_dir.all_paths():
            if not path.is_file:
                continue
            if path.name.endswith(".so"):
                detected_dep_requirements |= PERL_DEP_XS_MODULE
            elif path.name.endswith(".pm"):
                detected_dep_requirements |= pm_mode

    rewritten_shline = f"#! {DESIRED_PERL_SHBANG_CMD}\n"

    for path, children in fs_root.walk():
        if path.path == "./usr/share/doc":
            children.clear()
            continue
        if (
            not path.is_file
            or not path.has_fs_path
            or not ((path.mode & 0o0111) or path.name.endswith(".pl"))
        ):
            continue

        with path.open(byte_io=True) as fd:
            shbline = fd.read(32)
            m = PERL_SHBANG_RE.search(shbline)
            if not m:
                continue
            detected_dep_requirements |= PERL_DEP_PROGRAM
            perl = m.group(1).strip().decode("utf-8")

            if perl != DESIRED_PERL_SHBANG_CMD:
                _replace_shbang_line(path, rewritten_shline)

    if not detected_dep_requirements:
        return
    dpackage = "perl"
    # FIXME: Currently, dh_perl supports perl-base via manual toggle.

    dependency = dpackage
    if not (detected_dep_requirements & PERL_DEP_MA_ANY_INCOMPATIBLE_TYPES):
        dependency += ":any"

    if detected_dep_requirements & PERL_DEP_XS_MODULE:
        dpkg_version = _dpkg_perl_version(dpackage)
        dependency += f" (>= {dpkg_version})"
    substvars.add_dependency("perl:Depends", dependency)

    if detected_dep_requirements & (PERL_DEP_XS_MODULE | PERL_DEP_ARCH_PM_MODULE):
        substvars.add_dependency("perl:Depends", perlxs_api_dependency())


def _replace_shbang_line(path: FSPath, new_shbang_line: str) -> None:
    if not new_shbang_line.startswith("#!"):
        raise ValueError("The shbang line must start with '#!'")
    if not new_shbang_line.endswith("\n"):
        new_shbang_line += "\n"
    parent_dir = path.parent_dir
    mode = path.mode
    name = path.name
    with path.open(byte_io=True) as rfd:
        original_shline = rfd.readline()
        if not original_shline.startswith(b"#!"):
            raise ValueError(
                f'The provided path "{path.path}" does not start with a shbang line!?'
            )
        mtime = path.mtime
        path.unlink()
        del path  # path is detached, so we should stop using it.
        with parent_dir.add_file(name, mode=mode, mtime=mtime) as new_path, open(
            new_path.fs_path, "wb"
        ) as wfd:
            wfd.write(new_shbang_line.encode("utf-8"))
            shutil.copyfileobj(rfd, wfd)


def usr_local_transformation(dctrl: BinaryPackage, fs_root: FSPath) -> None:
    path = fs_root.lookup("./usr/local")
    if path and any(path.iterdir):
        # There are two key issues:
        #  1) Getting the generated maintscript carried on to the final maintscript
        #  2) Making sure that manifest created directories do not trigger the "unused error".
        _error(
            f"Replacement of /usr/local paths is currently not supported in debputy (triggered by: {dctrl.name})."
        )


def _find_and_analyze_systemd_service_files(
    fs_root: VirtualPath,
    systemd_service_dir: Literal["system", "user"],
) -> Iterable[VirtualPath]:
    service_dirs = [
        f"./usr/lib/systemd/{systemd_service_dir}",
        f"./lib/systemd/{systemd_service_dir}",
    ]
    aliases: Dict[str, List[str]] = collections.defaultdict(list)
    seen = set()
    all_files = []

    for d in service_dirs:
        system_dir = fs_root.lookup(d)
        if not system_dir:
            continue
        for child in system_dir.iterdir:
            if child.is_symlink:
                dest = os.path.basename(child.readlink())
                aliases[dest].append(child.name)
            elif child.is_file and child.name not in seen:
                seen.add(child.name)
                all_files.append(child)

    return all_files


def detect_systemd_system_service_files(
    dctrl: BinaryPackage,
    fs_root: VirtualPath,
) -> None:
    for service_file in _find_and_analyze_systemd_service_files(fs_root, "system"):
        _error(
            f'Sorry, systemd system services files are not supported at the moment (saw "{service_file.path}"'
            f" in {dctrl.name})"
        )


def detect_systemd_user_service_files(
    dctrl: BinaryPackage,
    fs_root: VirtualPath,
) -> None:
    for service_file in _find_and_analyze_systemd_service_files(fs_root, "user"):
        _error(
            f'Sorry, systemd user services files are not supported at the moment (saw "{service_file.path}"'
            f" in {dctrl.name})"
        )


@contextlib.contextmanager
def _open_maybe_gzip(path: FSPath) -> Iterator[IO[bytes]]:
    if path.name.endswith(".gz"):
        with gzip.GzipFile(path.fs_path, "rb") as fd:
            yield fd
    else:
        with path.open(byte_io=True) as fd:
            yield fd


_SO_LINK_RE = re.compile(rb"[.]so\s+(.*)\s*")


def _detect_so_link(path: FSPath) -> Optional[str]:
    so_link_re = _SO_LINK_RE
    with _open_maybe_gzip(path) as fd:
        for line in fd:
            m = so_link_re.search(line)
            if m:
                return m.group(1)


def _replace_with_symlink(path: FSPath, so_link_target: str) -> None:
    adjusted_target = so_link_target
    parent_dir = path.parent_dir
    assert parent_dir is not None  # For the type checking
    if parent_dir.name == os.path.dirname(adjusted_target):
        # Avoid man8/../man8/foo links
        adjusted_target = os.path.basename(adjusted_target)
    elif "/" in so_link_target:
        # symlinks and so links have a different base directory when the link has a "/".
        # Adjust with an extra "../" to align the result
        adjusted_target = "../" + adjusted_target

    path.unlink()
    parent_dir.add_symlink(path.name, adjusted_target)


@functools.lru_cache(1)
def _has_man_recode() -> bool:
    # Ideally, we would just use shutil.which or something like that.
    # Unfortunately, in debhelper, we experienced problems with which
    # returning "yes" for a man tool that actually could not be run
    # on salsa CI.
    #
    # Therefore, we adopt the logic of dh_installman to run the tool
    # with --help to confirm it is not broken, because no one could
    # figure out what happened in the salsa CI and my life is still
    # too short to figure it out.
    try:
        subprocess.check_call(
            ["man-recode", "--help"],
            stdin=subprocess.DEVNULL,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            restore_signals=True,
        )
    except subprocess.CalledProcessError:
        return False
    return True


def process_manpages(fs_root: FSPath) -> None:
    man_dir = fs_root.lookup("./usr/share/man")
    if not man_dir:
        return

    re_encode = []
    for path in (p for p in man_dir.all_paths() if p.is_file and p.has_fs_path):
        size = path.stat().st_size
        if size == 0:
            continue
        so_link_target = None
        if size <= 1024:
            # debhelper has a 1024 byte guard on the basis that ".so file tend to be small".
            # That guard worked well for debhelper, so lets keep it for now on that basis alone.
            so_link_target = _detect_so_link(path)
        if so_link_target:
            _replace_with_symlink(path, so_link_target)
        else:
            re_encode.append(path)

    if not re_encode or not _has_man_recode():
        return

    with ExitStack() as manager:
        manpages = [
            manager.enter_context(p.replace_fs_path_content()) for p in re_encode
        ]
        static_cmd = ["man-recode", "--to-code", "UTF-8", "--suffix", ".encoded"]
        for cmd in xargs(static_cmd, manpages):
            _info(f"Ensuring manpages have utf-8 encoding via: {escape_shell(*cmd)}")
            try:
                subprocess.check_call(
                    cmd,
                    stdin=subprocess.DEVNULL,
                    restore_signals=True,
                )
            except subprocess.CalledProcessError:
                _error(
                    "The man-recode process failed. Please review the output of `man-recode` to understand"
                    " what went wrong."
                )
        for manpage in manpages:
            os.rename(f"{manpage}.encoded", manpage)


# Generally, this should match the release date of oldstable or oldoldstable
_DCH_PRUNE_CUT_OFF_DATE = datetime.date(2019, 7, 6)
_DCH_MIN_NUM_OF_ENTRIES = 4


def _prune_dch_file(
    package: BinaryPackage,
    path: FSPath,
    is_changelog: bool,
    keep_versions: Optional[Set[str]],
    *,
    trim: bool = True,
) -> Tuple[bool, Optional[Set[str]]]:
    # TODO: Process `d/changelog` once
    # Note we cannot assume that changelog_file is always `d/changelog` as you can have
    # per-package changelogs.
    with path.open() as fd:
        dch = Changelog(fd)
    shortened = False
    important_entries = 0
    binnmu_entries = []
    if is_changelog:
        kept_entries = []
        for block in dch:
            if block.other_pairs.get("binary-only", "no") == "yes":
                # Always keep binNMU entries (they are always in the top) and they do not count
                # towards our kept_entries limit
                binnmu_entries.append(block)
                continue
            entry_date = datetime.datetime.strptime(
                block.date, "%a, %d %b %Y %H:%M:%S %z"
            ).date()
            if (
                trim
                and entry_date < _DCH_PRUNE_CUT_OFF_DATE
                and important_entries >= _DCH_MIN_NUM_OF_ENTRIES
            ):
                shortened = True
                break
            # Match debhelper in incrementing after the check.
            important_entries += 1
            kept_entries.append(block)
    else:
        # The NEWS files should match the version for the dch to avoid lintian warnings.
        # If that means we remove all entries in the NEWS file, then we delete the NEWS
        # file (see #1021607)
        kept_entries = [b for b in dch if b.version in keep_versions]
        shortened = len(dch) > len(kept_entries)
        if shortened and not kept_entries:
            path.unlink()
            return True, None

    if not shortened and not binnmu_entries:
        return False, None

    parent_dir = path.parent_dir

    with path.replace_fs_path_content() as fs_path, open(
        fs_path, "wt", encoding="utf-8"
    ) as fd:
        for entry in kept_entries:
            fd.write(str(entry))

        if is_changelog and shortened:
            # For changelog (rather than NEWS) files, add a note about how to
            # get the full version.
            msg = textwrap.dedent(
                f"""\
                # Older entries have been removed from this changelog.
                # To read the complete changelog use `apt changelog {package.name}`.
                """
            )
            fd.write(msg)

    if binnmu_entries:
        if package.is_arch_all:
            _error(
                f"The package {package.name} is architecture all, but it is built during a binNMU. A binNMU build"
                " must not include architecture all packages"
            )

        with parent_dir.add_file(
            f"{path.name}.{package.resolved_architecture}"
        ) as binnmu_changelog, open(
            binnmu_changelog.fs_path,
            "wt",
            encoding="utf-8",
        ) as binnmu_fd:
            for entry in binnmu_entries:
                binnmu_fd.write(str(entry))

    if not shortened:
        return False, None
    return True, {b.version for b in kept_entries}


def fixup_debian_changelog_and_news_file(
    dctrl: BinaryPackage,
    fs_root: FSPath,
    is_native: bool,
    build_env: DebOptionsAndProfiles,
) -> None:
    doc_dir = fs_root.lookup(f"./usr/share/doc/{dctrl.name}")
    if not doc_dir:
        return
    changelog = doc_dir.get("changelog.Debian")
    if changelog and is_native:
        changelog.name = "changelog"
    elif is_native:
        changelog = doc_dir.get("changelog")

    trim = False if "notrimdch" in build_env.deb_build_options else True

    kept_entries = None
    pruned_changelog = False
    if changelog and changelog.has_fs_path:
        pruned_changelog, kept_entries = _prune_dch_file(
            dctrl, changelog, True, None, trim=trim
        )

    if not trim:
        return

    news_file = doc_dir.get("NEWS.Debian")
    if news_file and news_file.has_fs_path and pruned_changelog:
        _prune_dch_file(dctrl, news_file, False, kept_entries)


_UPSTREAM_CHANGELOG_SOURCE_DIRS = [
    ".",
    "doc",
    "docs",
]
_UPSTREAM_CHANGELOG_NAMES = {
    # The value is a priority to match the debhelper order.
    #  - The suffix weights heavier than the basename (because that is what debhelper did)
    #
    # We list the name/suffix in order of priority in the code. That makes it easier to
    # see the priority directly, but it gives the "lowest" value to the most important items
    f"{n}{s}": (sw, nw)
    for (nw, n), (sw, s) in itertools.product(
        enumerate(["changelog", "changes", "history"], start=1),
        enumerate(["", ".txt", ".md", ".rst"], start=1),
    )
}
_NONE_TUPLE = (None, (0, 0))


def _detect_upstream_changelog(names: Iterable[str]) -> Optional[str]:
    matches = []
    for name in names:
        match_priority = _UPSTREAM_CHANGELOG_NAMES.get(name.lower())
        if match_priority is not None:
            matches.append((name, match_priority))
    return min(matches, default=_NONE_TUPLE, key=operator.itemgetter(1))[0]


def install_upstream_changelog(
    dctrl_bin: BinaryPackage, fs_root: FSPath, source_fs_root: str
) -> None:
    bdir = fs_root.lookup(f"./usr/share/doc/{dctrl_bin.name}")
    if bdir:
        if bdir.get("changelog") or bdir.get("changelog.gz"):
            # Upstream's build system already provided the changelog with the correct name.
            # Accept that as the canonical one.
            return
        upstream_changelog = _detect_upstream_changelog(
            p.name
            for p in bdir.iterdir
            if p.is_file and p.has_fs_path and p.stat().st_size > 0
        )
        if upstream_changelog:
            p = fs_root.lookup(upstream_changelog)
            assert p is not None  # Mostly as a typing hint
            p.name = "changelog"
            return
    for dirname in _UPSTREAM_CHANGELOG_SOURCE_DIRS:
        dir_path = (
            os.path.join(source_fs_root, dirname) if dirname != "." else source_fs_root
        )
        if not os.path.isdir(dir_path):
            continue
        changelog_name = _detect_upstream_changelog(
            sorted(f.name for f in os.scandir(dir_path) if f.is_file())
        )
        if changelog_name:
            bdir.insert_file_from_fs_path(
                "changelog",
                os.path.join(dir_path, changelog_name),
                require_copy=False,
            )
            break


def apply_compression(
    dctrl: BinaryPackage, target_doc_package: BinaryPackage, fs_root: FSPath
) -> None:
    # TODO: Support hardlinks
    compressed_files: Dict[str, str] = {}
    for path in find_compressable_paths(dctrl, target_doc_package, fs_root):
        parent_dir = path.parent_dir
        with parent_dir.add_file(f"{path.name}.gz", mtime=path.mtime) as new_file, open(
            new_file.fs_path, "wb"
        ) as fd:
            try:
                subprocess.check_call(["gzip", "-9nc", path.fs_path], stdout=fd)
            except subprocess.CalledProcessError:
                full_command = f"gzip -9nc {escape_shell(path.fs_path)} > {escape_shell(new_file.fs_path)}"
                _error(
                    f"The compression of {path.path} failed. Please review the error message from gzip to"
                    f" understand what went wrong.  Full command was: {full_command}"
                )
            compressed_files[path.path] = new_file.path
        del parent_dir[path.name]

    all_remaining_symlinks = {p.path: p for p in fs_root.all_paths() if p.is_symlink}
    changed = True
    while changed:
        changed = False
        remaining: List[FSPath] = list(all_remaining_symlinks.values())
        for symlink in remaining:
            target = symlink.readlink()
            dir_target, basename_target = os.path.split(target)
            new_basename_target = f"{basename_target}.gz"
            dir_path = fs_root.lookup(dir_target)
            if (
                not dir_path
                or basename_target in dir_path
                or new_basename_target not in dir_path
            ):
                continue
            del all_remaining_symlinks[symlink.path]
            changed = True

            new_link_name = (
                f"{symlink.name}.gz"
                if not symlink.name.endswith(".gz")
                else symlink.name
            )
            symlink.parent_dir.add_symlink(
                new_link_name, os.path.join(dir_target, new_basename_target)
            )
            symlink.unlink()


@dataclasses.dataclass(slots=True)
class _ElfInfo:
    path: FSPath
    fs_path: str
    is_stripped: Optional[bool] = None
    build_id: Optional[str] = None
    dbgsym: Optional[FSPath] = None


def _elf_static_lib_walk_filter(
    fs_path: VirtualPath,
    children: List[VirtualPath],
) -> bool:
    if fs_path.name == ".build-id" and fs_path.parent_dir.name == "debug":
        children.clear()
        return False
    # Deal with some special cases, where certain files are not supposed to be stripped in a given directory
    if "debug/" in fs_path.path or fs_path.name.endswith("debug/"):
        # FIXME: We need a way to opt out of this per #468333/#1016122
        for so_file in (f for f in list(children) if f.name.endswith(".so")):
            children.remove(so_file)
    if "/guile/" in fs_path.path or fs_path.name == "guile":
        for go_file in (f for f in list(children) if f.name.endswith(".go")):
            children.remove(go_file)
    return True


@contextlib.contextmanager
def _all_elf_files(fs_root: FSPath) -> Iterator[Dict[str, _ElfInfo]]:
    all_elf_files = find_all_elf_files(
        fs_root,
        walk_filter=_elf_static_lib_walk_filter,
    )
    if not all_elf_files:
        yield {}
        return
    with ExitStack() as cm_stack:
        resolved = (
            (p, cm_stack.enter_context(p.replace_fs_path_content()))
            for p in all_elf_files
        )
        elf_info = {
            fs_path: _ElfInfo(
                path=assume_not_none(fs_root.lookup(detached_path.path)),
                fs_path=fs_path,
            )
            for detached_path, fs_path in resolved
        }
        _resolve_build_ids(elf_info)
        yield elf_info


def _find_all_static_libs(
    fs_root: FSPath,
) -> Iterator[str]:
    for path, children in fs_root.walk():
        # Matching the logic of dh_strip for now.
        if not _elf_static_lib_walk_filter(path, children):
            continue
        if not path.is_file:
            continue
        if path.name.startswith("lib") and path.name.endswith("_g.a"):
            # _g.a are historically ignored. I do not remember why, but guessing the "_g" is
            # a encoding of gcc's -g parameter into the filename (with -g meaning "I want debug
            # symbols")
            continue
        if not path.has_fs_path:
            continue
        with path.open(byte_io=True) as fd:
            magic = fd.read(8)
            if magic not in (b"!<arch>\n", b"!<thin>\n"):
                continue
            # Maybe we should see if the first file looks like an index file.
            # Three random .a samples suggests the index file is named "/"
            # Not sure if we should skip past it and then do the ELF check or just assume
            # that "index => static lib".
            data = fd.read(1024 * 1024)
            if b"\0" not in data and ELF_MAGIC not in data:
                continue
        yield path


@contextlib.contextmanager
def _all_static_libs(fs_root: FSPath) -> Iterator[List[str]]:
    all_static_libs = list(_find_all_static_libs(fs_root))
    if not all_static_libs:
        yield []
        return
    with ExitStack() as cm_stack:
        resolved = [
            cm_stack.enter_context(p.replace_fs_path_content()) for p in all_static_libs
        ]
        yield resolved


_FILE_BUILD_ID_RE = re.compile(rb"BuildID(?:\[\S+\])?=([A-Fa-f0-9]+)")


def _resolve_build_ids(elf_info: Dict[str, _ElfInfo]) -> None:
    # FIXME: xargs
    static_cmd = ["file", "-00", "-N"]
    if detect_fakeroot():
        static_cmd.append("--no-sandbox")

    for cmd in xargs(static_cmd, (i.fs_path for i in elf_info.values())):
        _info(f"Looking up build-ids via: {escape_shell(*cmd)}")
        output = subprocess.check_output(cmd)

        # Trailing "\0" gives an empty element in the end when splitting, so strip it out
        lines = output.rstrip(b"\0").split(b"\0")

        for fs_path_b, verdict in grouper(lines, 2, incomplete="strict"):
            fs_path = fs_path_b.decode("utf-8")
            info = elf_info[fs_path]
            info.is_stripped = b"not stripped" not in verdict
            m = _FILE_BUILD_ID_RE.search(verdict)
            if m:
                info.build_id = m.group(1).decode("utf-8")


def _make_debug_file(
    objcopy: str, fs_path: str, build_id: str, dbgsym_fs_root: FSPath
) -> FSPath:
    dbgsym_dirname = f"./usr/lib/debug/.build-id/{build_id[0:2]}/"
    dbgsym_basename = f"{build_id[2:]}.debug"
    dbgsym_dir = dbgsym_fs_root.mkdirs(dbgsym_dirname)
    if dbgsym_basename in dbgsym_dir:
        return dbgsym_dir[dbgsym_basename]
    # objcopy is a pain and includes the basename verbatim when you do `--add-gnu-debuglink` without having an option
    # to overwrite the physical basename.  So we have to ensure that the physical basename matches the installed
    # basename.
    with dbgsym_dir.add_file(
        dbgsym_basename,
        exist_ok=False,
        fs_basename_matters=True,
        subdir_key="dbgsym-build-ids",
    ) as dbgsym:
        try:
            subprocess.check_call(
                [
                    objcopy,
                    "--only-keep-debug",
                    "--compress-debug-sections",
                    fs_path,
                    dbgsym.fs_path,
                ]
            )
        except subprocess.CalledProcessError:
            full_command = (
                f"{objcopy} --only-keep-debug --compress-debug-sections"
                f" {escape_shell(fs_path, dbgsym.fs_path)}"
            )
            _error(
                f"Attempting to create a .debug file failed. Please review the error message from {objcopy} to"
                f" understand what went wrong.  Full command was: {full_command}"
            )
    return dbgsym


def _strip_binary(strip: str, options: List[str], paths: Iterable[str]) -> None:
    # We assume the paths are obtained via `p.replace_fs_path_content()`,
    # which is the case at the time of written and should remain so forever.
    it = iter(paths)
    first = next(it, None)
    if first is None:
        return
    static_cmd = [strip]
    static_cmd.extend(options)

    for cmd in xargs(static_cmd, itertools.chain((first,), (f for f in it))):
        _info(f"Removing unnecessary ELF debug info via: {escape_shell(*cmd)}")
        try:
            subprocess.check_call(
                cmd,
                stdin=subprocess.DEVNULL,
                restore_signals=True,
            )
        except subprocess.CalledProcessError:
            _error(
                f"Attempting to remove ELF debug info failed. Please review the error from {strip} above"
                f" understand what went wrong."
            )


def _attach_debug(objcopy: str, elf_binary: FSPath, dbgsym: FSPath) -> None:
    with dbgsym.replace_fs_path_content() as dbgsym_fs_path:
        cmd = [objcopy, "--add-gnu-debuglink", dbgsym_fs_path, elf_binary.fs_path]
        print_command(*cmd)
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError:
            _error(
                f"Attempting to attach ELF debug link to ELF binary failed. Please review the error from {objcopy}"
                f" above understand what went wrong."
            )


def _run_dwz(
    dctrl: BinaryPackage,
    dbgsym_fs_root: FSPath,
    unstripped_elf_info: List[_ElfInfo],
) -> None:
    if not unstripped_elf_info or dctrl.is_udeb:
        return
    dwz_cmd = ["dwz"]
    dwz_ma_dir_name = f"usr/lib/debug/.dwz/{dctrl.deb_multiarch}"
    dwz_ma_basename = f"{dctrl.name}.debug"
    multifile = f"{dwz_ma_dir_name}/{dwz_ma_basename}"
    build_time_multifile = None
    if len(unstripped_elf_info) > 1:
        fs_content_dir = generated_content_dir()
        fd, build_time_multifile = mkstemp(suffix=dwz_ma_basename, dir=fs_content_dir)
        os.close(fd)
        dwz_cmd.append(f"-m{build_time_multifile}")
        dwz_cmd.append(f"-M/{multifile}")

    # TODO: configuration for disabling multi-file and tweaking memory limits

    dwz_cmd.extend(e.fs_path for e in unstripped_elf_info)

    _info(f"Deduplicating ELF debug info via: {escape_shell(*dwz_cmd)}")
    try:
        subprocess.check_call(dwz_cmd)
    except subprocess.CalledProcessError:
        _error(
            "Attempting to deduplicate ELF info via dwz failed. Please review the output from dwz above"
            " to understand what went wrong."
        )
    if build_time_multifile is not None and os.stat(build_time_multifile).st_size > 0:
        dwz_dir = dbgsym_fs_root.mkdirs(dwz_ma_dir_name)
        dwz_dir.insert_file_from_fs_path(
            dwz_ma_basename,
            build_time_multifile,
            mode=0o644,
            require_copy=False,
            follow_symlinks=False,
        )


def relocate_dwarves_into_dbgsym_packages(
    dctrl: BinaryPackage,
    package_fs_root: FSPath,
    dbgsym_fs_root: FSPath,
) -> List[str]:
    # FIXME: hardlinks
    with _all_static_libs(package_fs_root) as all_static_files:
        if all_static_files:
            strip = dctrl.cross_command("strip")
            _strip_binary(
                strip,
                [
                    "--strip-debug",
                    "--remove-section=.comment",
                    "--remove-section=.note",
                    "--enable-deterministic-archives",
                    "-R",
                    ".gnu.lto_*",
                    "-R",
                    ".gnu.debuglto_*",
                    "-N",
                    "__gnu_lto_slim",
                    "-N",
                    "__gnu_lto_v1",
                ],
                all_static_files,
            )

    with _all_elf_files(package_fs_root) as all_elf_files:
        if not all_elf_files:
            return []
        objcopy = dctrl.cross_command("objcopy")
        strip = dctrl.cross_command("strip")
        unstripped_elf_info = list(
            e for e in all_elf_files.values() if not e.is_stripped
        )

        _run_dwz(dctrl, dbgsym_fs_root, unstripped_elf_info)

        for elf_info in unstripped_elf_info:
            elf_info.dbgsym = _make_debug_file(
                objcopy,
                elf_info.fs_path,
                elf_info.build_id,
                dbgsym_fs_root,
            )

        # Note: When run strip, we do so also on already stripped ELF binaries because that is what debhelper does!
        # Executables (defined by mode)
        _strip_binary(
            strip,
            ["--remove-section=.comment", "--remove-section=.note"],
            (i.fs_path for i in all_elf_files.values() if i.path.mode & 0o111),
        )

        # Libraries (defined by mode)
        _strip_binary(
            strip,
            ["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"],
            (i.fs_path for i in all_elf_files.values() if not (i.path.mode & 0o111)),
        )

        for elf_info in unstripped_elf_info:
            _attach_debug(objcopy, elf_info.path, elf_info.dbgsym)

        # Set for uniqueness
        all_debug_info = sorted({i.build_id for i in unstripped_elf_info})

    dbgsym_doc_dir = dbgsym_fs_root.mkdirs("./usr/share/doc/")
    dbgsym_doc_dir.add_symlink(f"{dctrl.name}-dbgsym", dctrl.name)
    return all_debug_info


def setup_control_files(
    binary_package_data: BinaryPackageData,
    manifest: HighLevelManifest,
    related_udeb_package_data: Optional[BinaryPackageData],
    dbgsym_fs_root: FSPath,
    dbgsym_ids: List[str],
) -> None:
    binary_package = assume_not_none(binary_package_data.binary_package)
    control_output_dir = binary_package_data.control_output_dir
    fs_root = binary_package_data.fs_root
    soname_info_list = assume_not_none(binary_package_data.shared_libraries)
    package_state = manifest.package_state_for(binary_package.name)

    generated_triggers = set()
    plugin_provided_triggers: Dict[
        Tuple[DpkgTriggerType, str], PluginProvidedTrigger
    ] = {}
    metadata_maintscript_detectors = (
        manifest.plugin_provided_feature_set.metadata_maintscript_detectors
    )
    package_metadata_context = MetadataDetectorPackageContextProvider(
        binary_package,
        related_udeb_package_data.binary_package if related_udeb_package_data else None,
        # FIXME: source_package
    )
    state = manifest.package_state_for(binary_package.name)

    if not binary_package.is_arch_all:
        udeb_package_name = (
            related_udeb_package_data.binary_package.name
            if related_udeb_package_data
            else None
        )
        compute_shlibs(
            binary_package,
            control_output_dir,
            fs_root,
            manifest,
            soname_info_list,
            udeb_package_name,
            generated_triggers,
            state.reserved_packager_provided_files,
        )

    substvars = binary_package_data.substvars

    maintscript_snippets = collections.defaultdict(
        MaintscriptSnippetContainer,
        **{k: v.copy() for k, v in state.maintscript_snippets.items()},
    )
    plugin_detector_definition: MetadataOrMaintscriptDetector
    for plugin_detector_definition in itertools.chain.from_iterable(
        metadata_maintscript_detectors.values()
    ):
        if not plugin_detector_definition.applies_to(binary_package):
            continue
        ctrl = BinaryCtrlAccessorProvider(
            plugin_detector_definition.plugin_metadata,
            package_metadata_context,
            plugin_detector_definition.detector_id,
            plugin_provided_triggers,
            substvars,
            maintscript_snippets,
            state.substitution,
        )
        plugin_detector_definition.run_detector(fs_root, ctrl, package_metadata_context)

    if binary_package.is_udeb:
        _generate_control_files(
            binary_package,
            package_state,
            control_output_dir,
            fs_root,
            substvars,
            # We never built udebs due to #797391, so skip over this information,
            # when creating the udeb
            None,
            None,
        )
        # FIXME: udeb scripts
        for script in ["postinst"]:
            _generate_snippet(control_output_dir, script, maintscript_snippets)
        return

    for script in STD_CONTROL_SCRIPTS:
        _generate_snippet(control_output_dir, script, maintscript_snippets)

    if generated_triggers:
        dest_file = os.path.join(control_output_dir, "triggers")
        with open(dest_file, "at", encoding="utf-8") as fd:
            fd.writelines(f"{t}\n" for t in generated_triggers)
            os.chmod(fd.fileno(), 0o644)
    install_or_generate_conffiles(binary_package, control_output_dir, fs_root)
    _generate_control_files(
        binary_package,
        package_state,
        control_output_dir,
        fs_root,
        substvars,
        dbgsym_fs_root,
        dbgsym_ids,
    )


def _generate_snippet(
    control_output_dir: str,
    script: str,
    maintscript_snippets: Dict[str, MaintscriptSnippetContainer],
) -> None:
    debputy_snippets = maintscript_snippets.get(script)
    if debputy_snippets is None:
        return
    reverse = script in ("prerm", "postrm")
    snippets = [
        debputy_snippets.generate_snippet(reverse=reverse),
        debputy_snippets.generate_snippet(snippet_order="service", reverse=reverse),
    ]
    if reverse:
        snippets = reversed(snippets)
    full_content = "".join(f"{s}\n" for s in filter(None, snippets))
    if not full_content:
        return
    filename = os.path.join(control_output_dir, script)
    with open(filename, "wt") as fd:
        fd.write(full_content)
        os.chmod(fd.fileno(), 0o755)


def _add_conffiles(conffiles_dest: str, conffile_matches: Iterable[FSPath]) -> None:
    with open(conffiles_dest, "at") as fd:
        for conffile_match in conffile_matches:
            conffile = conffile_match.absolute
            assert conffile_match.is_file
            fd.write(f"{conffile}\n")
    if os.stat(conffiles_dest).st_size == 0:
        os.utime(conffiles_dest)


def _ensure_base_substvars_defined(substvars: Substvars) -> None:
    for substvar in ("misc:Depends", "misc:Pre-Depends"):
        if substvar not in substvars:
            substvars[substvar] = ""


def _compute_installed_size(fs_root: FSPath) -> int:
    """Emulate dpkg-gencontrol's code for computing the default Installed-Size"""
    size_in_kb = 0
    hard_links = set()
    for path in fs_root.all_paths():
        if not path.is_dir and path.has_fs_path:
            st = path.stat()
            if st.st_nlink > 1:
                hl_key = (st.st_dev, st.st_ino)
                if hl_key in hard_links:
                    continue
                hard_links.add(hl_key)
            path_size = (st.st_size + 1023) // 1024
        elif path.is_symlink:
            path_size = (len(path.readlink()) + 1023) // 1024
        else:
            path_size = 1
        size_in_kb += path_size
    return size_in_kb


def _generate_dbgsym_control_file_if_relevant(
    binary_package: BinaryPackage,
    dbgsym_root_dir: str,
    substvars_file: str,
    dbgsym_ids: str,
    multi_arch: Optional[str],
    extra_common_params: Sequence[str],
) -> None:
    section = binary_package.archive_section
    component = ""
    extra_params = []
    if section is not None and "/" in section and not section.startswith("main/"):
        component = section.split("/", 1)[1] + "/"
    if multi_arch != "same":
        extra_params.append("-UMulti-Arch")
    extra_params.append("-UReplaces")
    extra_params.append("-UBreaks")
    extra_params.extend(extra_common_params)

    ensure_dir(os.path.join(dbgsym_root_dir, "DEBIAN"))
    package = binary_package.name
    dpkg_cmd = [
        "dpkg-gencontrol",
        f"-p{package}",
        "-ldebian/changelog",
        f"-T{substvars_file}",
        f"-P{dbgsym_root_dir}",
        f"-DPackage={package}-dbgsym",
        "-DDepends=" + package + " (= ${binary:Version})",
        f"-DDescription=debug symbols for {package}",
        f"-DSection={component}debug",
        f"-DBuild-Ids={dbgsym_ids}",
        "-UPre-Depends",
        "-URecommends",
        "-USuggests",
        "-UEnhances",
        "-UProvides",
        "-UEssential",
        "-UConflicts",
        "-DPriority=optional",
        "-UHomepage",
        "-UImportant",
        "-UBuilt-Using",
        "-DAuto-Built-Package=debug-symbols",
        "-UProtected",
        *extra_params,
    ]
    print_command(*dpkg_cmd)
    try:
        subprocess.check_call(dpkg_cmd)
    except subprocess.CalledProcessError:
        _error(
            f"Attempting to generate DEBIAN/control file for {package}-dbgsym failed. Please review the output from "
            " dpkg-gencontrol above to understand what went wrong."
        )
    os.chmod(os.path.join(dbgsym_root_dir, "DEBIAN", "control"), 0o644)


def _all_parent_directories_of(directories: Iterable[str]) -> Set[str]:
    result = {"."}
    for path in directories:
        current = os.path.dirname(path)
        while current and current not in result:
            result.add(current)
            current = os.path.dirname(current)
    return result


def _auto_compute_multi_arch(
    binary_package: BinaryPackage,
    control_output_dir: str,
    fs_root: FSPath,
) -> Optional[str]:
    resolved_arch = binary_package.resolved_architecture
    if resolved_arch == "all":
        return None
    if any(
        script
        for script in ALL_CONTROL_SCRIPTS
        if os.path.isfile(os.path.join(control_output_dir, script))
    ):
        return None

    resolved_multiarch = binary_package.deb_multiarch
    assert resolved_arch != "all"
    acceptable_no_descend_paths = {
        f"./usr/lib/{resolved_multiarch}",
        f"./usr/include/{resolved_multiarch}",
    }
    acceptable_files = {
        f"./usr/share/doc/{binary_package.name}/{basename}"
        for basename in (
            "copyright",
            "changelog.gz",
            "changelog.Debian.gz",
            f"changelog.Debian.{resolved_arch}.gz",
            "NEWS.Debian",
            "NEWS.Debian.gz",
            "README.Debian",
            "README.Debian.gz",
        )
    }
    acceptable_intermediate_dirs = _all_parent_directories_of(
        itertools.chain(acceptable_no_descend_paths, acceptable_files)
    )

    for fs_path, children in fs_root.walk():
        path = fs_path.path
        if path in acceptable_no_descend_paths:
            children.clear()
            continue
        if path in acceptable_intermediate_dirs or path in acceptable_files:
            continue
        return None

    return "same"


def _t64_migration_substvar(
    binary_package: BinaryPackage,
    control_output_dir: str,
    substvars: Substvars,
) -> None:
    name = binary_package.name
    if not name.startswith("lib") or not name.endswith("t64"):
        return

    arch = binary_package.resolved_architecture
    arch_bits = binary_package.package_deb_architecture_variable("ARCH_BITS")

    if arch != "i386" and arch_bits == "32":
        return

    if not any(
        os.path.isfile(os.path.join(control_output_dir, n))
        for n in ["symbols", "shlibs"]
    ):
        return

    compat_name = name[:-3]
    substvars.add_dependency("t64:Provides", f"{compat_name} (= ${{binary:Version}})")


def _generate_control_files(
    binary_package: BinaryPackage,
    package_state: PackageTransformationDefinition,
    control_output_dir: str,
    fs_root: FSPath,
    substvars: FlushableSubstvars,
    dbgsym_root_fs: Optional[FSPath],
    dbgsym_build_ids: Optional[List[str]],
) -> None:
    package = binary_package.name
    extra_common_params = []
    extra_params_specific = []
    _ensure_base_substvars_defined(substvars)
    if "Installed-Size" not in substvars:
        # Pass it via cmd-line to make it more visible that we are providing the
        # value.  It also prevents the dbgsym package from picking up this value.
        ctrl_fs_root = build_fs_from_root_dir(control_output_dir)
        total_size = _compute_installed_size(fs_root) + _compute_installed_size(
            ctrl_fs_root
        )
        extra_params_specific.append(f"-VInstalled-Size={total_size}")

    ma_value = binary_package.fields.get("Multi-Arch")
    if not binary_package.is_udeb and ma_value is None:
        ma_value = _auto_compute_multi_arch(binary_package, control_output_dir, fs_root)
        if ma_value is not None:
            _info(
                f'The package "{binary_package.name}" looks like it should be "Multi-Arch: {ma_value}" based'
                ' on the contents and there is no explicit "Multi-Arch" field. Setting the Multi-Arch field'
                ' accordingly in the binary.  If this auto-correction is wrong, please  add "Multi-Arch: no" to the'
                ' relevant part of "debian/control" to disable this feature.'
            )
            extra_params_specific.append(f"-DMulti-Arch={ma_value}")
    elif ma_value == "no":
        extra_params_specific.append("-UMulti-Arch")

    dbgsym_root_dir = dhe_dbgsym_root_dir(binary_package)
    dbgsym_ids = " ".join(dbgsym_build_ids) if dbgsym_build_ids else ""
    if package_state.binary_version is not None:
        extra_common_params.append(f"-v{package_state.binary_version}")

    _t64_migration_substvar(binary_package, control_output_dir, substvars)

    with substvars.flush() as flushed_substvars:
        if dbgsym_root_fs is not None and any(
            f for f in dbgsym_root_fs.all_paths() if f.is_file
        ):
            _generate_dbgsym_control_file_if_relevant(
                binary_package,
                dbgsym_root_dir,
                flushed_substvars,
                dbgsym_ids,
                ma_value,
                extra_common_params,
            )
            generate_md5sums_file(
                os.path.join(dbgsym_root_dir, "DEBIAN"),
                dbgsym_root_fs,
            )
        elif dbgsym_ids:
            extra_common_params.append(f"-DBuild-Ids={dbgsym_ids}")
        ctrl_file = os.path.join(control_output_dir, "control")
        dpkg_cmd = [
            "dpkg-gencontrol",
            f"-p{package}",
            "-ldebian/changelog",
            f"-T{flushed_substvars}",
            f"-O{ctrl_file}",
            f"-P{control_output_dir}",
            *extra_common_params,
            *extra_params_specific,
        ]
        print_command(*dpkg_cmd)
        try:
            subprocess.check_call(dpkg_cmd)
        except subprocess.CalledProcessError:
            _error(
                f"Attempting to generate DEBIAN/control file for {package} failed. Please review the output from "
                " dpkg-gencontrol above to understand what went wrong."
            )
        os.chmod(ctrl_file, 0o644)

    if not binary_package.is_udeb:
        generate_md5sums_file(control_output_dir, fs_root)
