import collections
import contextlib
import dataclasses
import itertools
import os
from typing import (
    Tuple,
    Optional,
    Dict,
    TypeVar,
    Callable,
    List,
    Any,
    Union,
    Mapping,
    IO,
    Iterator,
    NotRequired,
    Annotated,
    Generic,
    Type,
    TypedDict,
    cast,
)

from debian.debian_support import DpkgArchTable

from debputy.highlevel_manifest import (
    HighLevelManifest,
    PackageTransformationDefinition,
    MutableYAMLManifest,
    MANIFEST_YAML,
)
from debputy.maintscript_snippet import (
    DpkgMaintscriptHelperCommand,
    MaintscriptSnippet,
    STD_CONTROL_SCRIPTS,
    MaintscriptSnippetContainer,
)
from debputy.packages import BinaryPackage
from debputy.path_matcher import MatchRuleType, ExactFileSystemPath, MatchRule
from debputy.substitution import Substitution
from debputy.transformation_rules import (
    TransformationRule,
    RemoveTransformationRule,
    MoveTransformationRule,
    CreateSymlinkPathTransformationRule,
    PathMetadataTransformationRule,
    CreateDirectoryTransformationRule,
)
from debputy.util import (
    _normalize_path,
    PKGVERSION_REGEX,
    PKGNAME_REGEX,
    escape_shell,
    active_profiles_match,
    debian_policy_normalize_symlink_target,
)
from debputy.util import _warn, _info
from ._deb_options_profiles import DebOptionsAndProfiles
from ._manifest_constants import *
from .architecture_support import DpkgArchitectureBuildProcessValuesTable
from .filesystem_scan import FSROOverlay
from .installations import InstallRule, PPFInstallRule
from .manifest_conditions import (
    ManifestCondition,
    ArchMatchManifestCondition,
    BuildProfileMatch,
)
from .manifest_parser.base_types import (
    DebputyParsedContentStandardConditional,
    FileSystemMode,
    FileSystemGroup,
    FileSystemOwner,
    DebputyParsedContent,
)
from .manifest_parser.declarative_parser import (
    DeclarativeInputParser,
    DebputyParseHint,
    parser_from_typed_dict,
)
from .manifest_parser.exceptions import ManifestParseException
from .manifest_parser.parser_data import ParserContextData
from .manifest_parser.util import AttributePath
from .packager_provided_files import detect_all_packager_provided_files
from .plugin.api import VirtualPath
from .plugin.api.impl_types import PluginProvidedFeatureSet

try:
    from Levenshtein import distance
except ImportError:

    def _detect_possible_typo(
        _d,
        _key,
        _attribute_parent_path: AttributePath,
        required: bool,
    ) -> None:
        if required:
            _info(
                "Install python3-levenshtein to have debputy try to detect typos in the manifest."
            )

else:

    def _detect_possible_typo(
        d,
        key,
        _attribute_parent_path: AttributePath,
        _required: bool,
    ) -> None:
        k_len = len(key)
        for actual_key in d:
            if abs(k_len - len(actual_key)) > 2:
                continue
            d = distance(key, actual_key)
            if d > 2:
                continue
            path = _attribute_parent_path.path
            ref = f'at "{path}"' if path else "at the manifest root level"
            _warn(
                f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
            )


PF = TypeVar("PF")
TP = TypeVar("TP")


@dataclasses.dataclass(slots=True, frozen=True)
class DispatchingParser(Generic[PF, TP]):
    parser: DeclarativeInputParser[PF]
    handler: Callable[[str, PF, AttributePath, ParserContextData], TP]


# TODO: Make this something plugins can hook into
_INSTALL_RULE_PARSERS: Dict[str, DispatchingParser[PF, InstallRule]] = {}
_TRANSFORMATION_RULE_PARSERS: Dict[str, DispatchingParser[PF, TransformationRule]] = {}
_DPKG_CONFFILE_MGNT_PARSERS: Dict[
    str, DispatchingParser[PF, DpkgMaintscriptHelperCommand]
] = {}


_DIP = DeclarativeInputParser[PF]
_DIP_HDL = Callable[[str, PF, AttributePath, ParserContextData], TP]


def _create_dispatchable_parser_hook(
    dispatch_table: Dict[str, DispatchingParser[PF, TP]],
) -> Callable[[_DIP], Callable[[_DIP_HDL], _DIP_HDL]]:
    def _annotation(parser: _DIP) -> Callable[[_DIP_HDL], _DIP_HDL]:
        def _func(func: _DIP_HDL) -> _DIP_HDL:
            parse_logic = DispatchingParser(parser, func)
            if parser.name in dispatch_table:
                raise ValueError(f'The key "{parser.name}" is already taken')
            dispatch_table[parser.name] = parse_logic
            for alias in parser.aliases:
                if alias in dispatch_table:
                    raise ValueError(f'The key "{alias}" is already taken')
                dispatch_table[alias] = parse_logic
            return func

        return _func

    return _annotation


install_rule_parser = _create_dispatchable_parser_hook(_INSTALL_RULE_PARSERS)
transformation_rule_parser = _create_dispatchable_parser_hook(
    _TRANSFORMATION_RULE_PARSERS
)
dpkg_conffile_parser = _create_dispatchable_parser_hook(_DPKG_CONFFILE_MGNT_PARSERS)


def _per_package_subst_variables(
    p: BinaryPackage,
    *,
    name: Optional[str] = None,
) -> Dict[str, str]:
    return {
        "PACKAGE": name if name is not None else p.name,
    }


def _with_alt_form(t: Type[TypedDict]):
    return Union[
        t,
        List[str],
        str,
    ]


class ManifestRootRule(DebputyParsedContent):
    manifest_version: str
    installations: NotRequired[Optional[List[Any]]]
    packages: NotRequired[Dict[str, Any]]


class _ModeOwnerBase(DebputyParsedContentStandardConditional):
    mode: NotRequired[FileSystemMode]
    owner: NotRequired[FileSystemOwner]
    group: NotRequired[FileSystemGroup]


class PathManifestSourceDictFormat(_ModeOwnerBase):
    path: Annotated[NotRequired[str], DebputyParseHint.target_attribute("paths")]
    paths: NotRequired[List[str]]
    recursive: NotRequired[bool]


class PathManifestRule(_ModeOwnerBase):
    paths: List[str]
    recursive: NotRequired[bool]


class EnsureDirectorySourceFormat(_ModeOwnerBase):
    path: Annotated[NotRequired[str], DebputyParseHint.target_attribute("paths")]
    paths: NotRequired[List[str]]


class EnsureDirectoryRule(_ModeOwnerBase):
    paths: List[str]


class CreateSymlinkRule(DebputyParsedContentStandardConditional):
    path: str
    target: str


class BinaryPackageRule(DebputyParsedContent):
    binary_version: NotRequired[str]
    transformations: NotRequired[List[Any]]
    conffile_management: NotRequired[List[Any]]


class DpkgConffileManagementRuleBase(DebputyParsedContent):
    prior_to_version: NotRequired[str]
    owning_package: NotRequired[str]


class DpkgRenameConffileRule(DpkgConffileManagementRuleBase):
    source: str
    target: str


class DpkgRemoveConffileRule(DpkgConffileManagementRuleBase):
    path: str


class TransformationMoveRuleSpec(DebputyParsedContentStandardConditional):
    source: str
    target: str


class TransformationRemoveRuleSpec(DebputyParsedContentStandardConditional):
    paths: List[str]


class TransformationRemoveRuleInputFormat(DebputyParsedContentStandardConditional):
    path: Annotated[NotRequired[str], DebputyParseHint.target_attribute("paths")]
    paths: NotRequired[List[str]]


class ParsedInstallRuleSourceFormat(DebputyParsedContentStandardConditional):
    sources: NotRequired[List[str]]
    source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")]
    into: Annotated[
        NotRequired[Union[str, List[str]]],
        DebputyParseHint.required_when_multi_binary(),
    ]
    dest_dir: NotRequired[str]
    as_: NotRequired[
        Annotated[
            str,
            DebputyParseHint.conflicts_with_source_attributes("sources", "dest_dir"),
            DebputyParseHint.target_attribute("install_as"),
        ]
    ]


class ParsedInstallRule(DebputyParsedContentStandardConditional):
    sources: List[str]
    into: NotRequired[List[BinaryPackage]]
    dest_dir: NotRequired[str]
    install_as: NotRequired[str]


class ParsedInstallDocOrExamplesRuleSourceFormat(
    DebputyParsedContentStandardConditional
):
    sources: NotRequired[List[str]]
    source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")]
    into: Annotated[
        NotRequired[Union[str, List[str]]],
        DebputyParseHint.required_when_multi_binary(),
    ]


class ParsedInstallDocOrExamplesRule(DebputyParsedContentStandardConditional):
    sources: List[str]
    into: NotRequired[List[BinaryPackage]]


class ParsedInstallManpageRule(DebputyParsedContentStandardConditional):
    sources: List[str]
    language: NotRequired[str]
    section: NotRequired[int]
    into: NotRequired[List[BinaryPackage]]


class ParsedInstallManpageRuleSourceFormat(DebputyParsedContentStandardConditional):
    sources: NotRequired[List[str]]
    source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")]
    language: NotRequired[str]
    section: NotRequired[int]
    into: Annotated[
        NotRequired[Union[str, List[str]]],
        DebputyParseHint.required_when_multi_binary(),
    ]


class ParsedInstallDiscardRuleSourceFormat(DebputyParsedContentStandardConditional):
    paths: NotRequired[List[str]]
    path: Annotated[NotRequired[str], DebputyParseHint.target_attribute("paths")]


class ParsedInstallDiscardRule(DebputyParsedContentStandardConditional):
    paths: List[str]


_INSTALL_PARSER = parser_from_typed_dict(
    MK_INSTALLATIONS_INSTALL,
    ParsedInstallRule,
    source_content=_with_alt_form(ParsedInstallRuleSourceFormat),
)
_INSTALL_DOCS_PARSER = parser_from_typed_dict(
    MK_INSTALLATIONS_INSTALL_DOCS,
    ParsedInstallDocOrExamplesRule,
    source_content=_with_alt_form(ParsedInstallDocOrExamplesRuleSourceFormat),
    aliases=[
        MK_INSTALLATIONS_INSTALL_EXAMPLES,
        "install-doc",
        "install-example",
    ],
)
_INSTALL_MAN_PARSER = parser_from_typed_dict(
    MK_INSTALLATIONS_INSTALL_MAN,
    ParsedInstallManpageRule,
    source_content=_with_alt_form(ParsedInstallManpageRuleSourceFormat),
)
_INSTALL_DISCARD_PARSER = parser_from_typed_dict(
    MK_INSTALLATIONS_DISCARD,
    ParsedInstallDiscardRule,
    source_content=_with_alt_form(ParsedInstallDiscardRuleSourceFormat),
)
_BINARY_PACKAGE_RULE_PARSER = parser_from_typed_dict(
    "binary-package-rule (value)", BinaryPackageRule
)
_DPKG_REMOVE_CONFFILE_PARSER = parser_from_typed_dict(
    "remove",
    DpkgRemoveConffileRule,
)
_DPKG_RENAME_CONFFILE_PARSER = parser_from_typed_dict(
    "rename",
    DpkgRenameConffileRule,
)
_TRANSFORMATION_MOVE = parser_from_typed_dict(
    "move",
    TransformationMoveRuleSpec,
)
_TRANSFORMATION_REMOVE = parser_from_typed_dict(
    "remove",
    TransformationRemoveRuleSpec,
    source_content=_with_alt_form(TransformationRemoveRuleInputFormat),
)
_TRANSFORMATION_CREATE_SYMLINK = parser_from_typed_dict(
    "create-symlink", CreateSymlinkRule
)
_TRANSFORMATION_PATH_METADATA_RULE_PARSER = parser_from_typed_dict(
    "path-metadata",
    PathManifestRule,
    source_content=PathManifestSourceDictFormat,
)
_TRANSFORMATION_ENSURE_DIR_RULE_PARSER = parser_from_typed_dict(
    "create-directories",
    EnsureDirectoryRule,
    source_content=_with_alt_form(EnsureDirectorySourceFormat),
)
_MANIFEST_ROOT_PARSER = parser_from_typed_dict(
    "root-parser", ManifestRootRule, allow_optional=True
)


# TODO: Should be configurable somehow
_SEARCH_DIRS = ["debian/tmp", "."]


@install_rule_parser(_INSTALL_PARSER)
def _install_rule_handler(
    _name: str,
    parsed_data: ParsedInstallRule,
    path: AttributePath,
    context: ParserContextData,
) -> InstallRule:
    sources_path = path["sources"]
    sources = [_parse_filename(p, sources_path) for p in parsed_data["sources"]]
    install_as = parsed_data.get("install_as")
    into = parsed_data.get("into")
    dest_dir = parsed_data.get("dest_dir")
    condition = parsed_data.get("when")
    if not into:
        into = [context.single_binary_package()]
    into = frozenset(into)
    if install_as is not None:
        assert len(sources) == 1
        assert dest_dir is None
        return InstallRule.install_as(
            sources[0],
            install_as,
            into,
            _SEARCH_DIRS,
            path.path,
            condition,
        )
    return InstallRule.install_dest(
        sources,
        dest_dir,
        into,
        _SEARCH_DIRS,
        path.path,
        condition,
    )


@install_rule_parser(_INSTALL_DOCS_PARSER)
def _install_docs_or_examples_rule_handler(
    name: str,
    parsed_data: ParsedInstallDocOrExamplesRule,
    path: AttributePath,
    context: ParserContextData,
) -> InstallRule:
    sources = parsed_data["sources"]
    into = parsed_data.get("into")
    if not into:
        into = [context.single_binary_package()]
    condition = parsed_data.get("when")
    into = frozenset(into)
    if name in (MK_INSTALLATIONS_INSTALL_EXAMPLES, "install-example"):
        install_rule = InstallRule.install_examples
    else:
        install_rule = InstallRule.install_doc
    return install_rule(
        sources,
        into,
        _SEARCH_DIRS,
        path.path,
        condition,
    )


@install_rule_parser(_INSTALL_MAN_PARSER)
def _install_man_rule_handler(
    _name: str,
    parsed_data: ParsedInstallManpageRule,
    attribute_path: AttributePath,
    context: ParserContextData,
) -> InstallRule:
    sources = parsed_data["sources"]
    language = parsed_data.get("language")
    section = parsed_data.get("section")

    if language is not None:
        is_lang_ok = language in (
            "C",
            "derive-from-basename",
            "derive-from-path",
        )

        if not is_lang_ok and len(language) == 2 and language.islower():
            is_lang_ok = True

        if (
            not is_lang_ok
            and len(language) == 5
            and language[2] == "_"
            and language[:2].islower()
            and language[3:].isupper()
        ):
            is_lang_ok = True

        if not is_lang_ok:
            raise ManifestParseException(
                f'The language attribute must in a 2-letter language code ("de"), a 5-letter language + dialect'
                f' code ("pt_BR"), "derive-from-basename", "derive-from-path", or omitted.  The problematic'
                f' definition is {attribute_path["language"]}'
            )

    if section is not None and (section < 1 or section > 10):
        raise ManifestParseException(
            f"The section attribute must in the range [1-9] or omitted.  The problematic definition is"
            f' {attribute_path["section"]}'
        )
    if section is None and any(s.endswith(".gz") for s in sources):
        raise ManifestParseException(
            "Sorry, compressed manpages are not supported without an explicit `section` definition at the moment."
            " This limitation may be removed in the future.  Problematic definition from"
            f' {attribute_path["sources"]}'
        )
    if any(s.endswith("/") for s in sources):
        raise ManifestParseException(
            'The install-man rule can only match non-directories.  Therefore, none of the sources can end with "/".'
            " as that implies the source is for a directory.  Problematic definition from"
            f' {attribute_path["sources"]}'
        )
    into = parsed_data.get("into")
    if not into:
        into = [context.single_binary_package()]
    condition = parsed_data.get("when")
    into = frozenset(into)
    return InstallRule.install_man(
        sources,
        into,
        section,
        language,
        _SEARCH_DIRS,
        attribute_path.path,
        condition,
    )


@install_rule_parser(_INSTALL_DISCARD_PARSER)
def _install_discard_rule_handler(
    _name: str,
    parsed_data: ParsedInstallDiscardRule,
    path: AttributePath,
    _context: ParserContextData,
) -> InstallRule:
    attr_path = path["paths"]
    paths = [_parse_filename(p, attr_path) for p in parsed_data["paths"]]
    condition = parsed_data.get("when")
    return InstallRule.discard_paths(
        paths,
        _SEARCH_DIRS,
        path.path,
        condition,
    )


@transformation_rule_parser(_TRANSFORMATION_MOVE)
def _transformation_move_handler(
    _name: str,
    parsed_data: TransformationMoveRuleSpec,
    path: AttributePath,
    context: ParserContextData,
) -> TransformationRule:
    source = parsed_data["source"]
    target = parsed_data["target"]
    condition = parsed_data.get("when")
    path.path_hint = source

    parser = cast("YAMLManifestParser", context)

    source_match = parser.parse_path_or_glob(source, path["source"])
    target_path = parser.normalize_path(target, path["target"]).path
    if (
        isinstance(source_match, ExactFileSystemPath)
        and source_match.path == target_path
    ):
        raise ManifestParseException(
            f"The transformation rule {path.path} requests a move of {source_match} to"
            f" {target_path}, which is the same path"
        )
    return MoveTransformationRule(
        source_match,
        target_path,
        target.endswith("/"),
        path,
        condition,
    )


@transformation_rule_parser(_TRANSFORMATION_REMOVE)
def _transformation_remove_handler(
    _name: str,
    parsed_data: TransformationRemoveRuleSpec,
    attribute_path: AttributePath,
    context: ParserContextData,
) -> TransformationRule:
    paths = parsed_data["paths"]
    attribute_path.path_hint = paths[0]
    parser = cast("YAMLManifestParser", context)

    match_rules = [parser.parse_path_or_glob(p, attribute_path) for p in paths]
    return RemoveTransformationRule(
        match_rules,
        attribute_path,
    )


@transformation_rule_parser(_TRANSFORMATION_CREATE_SYMLINK)
def _transformation_create_symlink(
    _name: str,
    parsed_data: CreateSymlinkRule,
    attribute_path: AttributePath,
    context: ParserContextData,
) -> TransformationRule:
    provided_link_path = parsed_data["path"]
    link_target_provided = parsed_data["target"]
    attribute_path.path_hint = provided_link_path
    parser = cast("YAMLManifestParser", context)
    link_dest = parser.normalize_path(provided_link_path, attribute_path["path"]).path
    try:
        link_target = debian_policy_normalize_symlink_target(
            link_dest, link_target_provided
        )
    except ValueError as e:
        raise AssertionError(
            "Debian Policy normalization should not raise ValueError here"
        ) from e

    condition = parsed_data.get("when")

    return CreateSymlinkPathTransformationRule(
        link_target,
        link_dest,
        attribute_path,
        condition,
    )


@transformation_rule_parser(_TRANSFORMATION_PATH_METADATA_RULE_PARSER)
def _transformation_path_metadata(
    _name: str,
    parsed_data: PathManifestRule,
    attribute_path: AttributePath,
    context: ParserContextData,
) -> TransformationRule:
    provided_paths = parsed_data["paths"]
    attribute_path.path_hint = provided_paths[0]
    owner = parsed_data.get("owner")
    group = parsed_data.get("group")
    mode = parsed_data.get("mode")
    recursive = parsed_data.get("recursive", False)
    parser = cast("YAMLManifestParser", context)
    match_rules = [
        parser.parse_path_or_glob(p, attribute_path["paths"]) for p in provided_paths
    ]

    if owner is None and group is None and mode is None:
        raise ManifestParseException(
            "At least one of owner, group and mode must be provided"
            f" in {attribute_path.path}"
        )
    condition = parsed_data.get("when")

    return PathMetadataTransformationRule(
        match_rules,
        owner,
        group,
        mode,
        recursive,
        attribute_path.path,
        condition,
    )


@transformation_rule_parser(_TRANSFORMATION_ENSURE_DIR_RULE_PARSER)
def _transformation_mkdirs(
    _name: str,
    parsed_data: PathManifestRule,
    attribute_path: AttributePath,
    context: ParserContextData,
) -> TransformationRule:
    provided_paths = parsed_data["paths"]
    attribute_path.path_hint = provided_paths[0]
    owner = parsed_data.get("owner")
    group = parsed_data.get("group")
    mode = parsed_data.get("mode")
    parser = cast("YAMLManifestParser", context)
    directories = [
        parser.normalize_path(p, attribute_path["paths"]).path for p in provided_paths
    ]

    condition = parsed_data.get("when")

    return CreateDirectoryTransformationRule(
        directories,
        owner,
        group,
        mode,
        attribute_path.path,
        condition,
    )


def _parse_filename(
    filename: str,
    attribute_path: AttributePath,
    *,
    allow_directories: bool = True,
) -> str:
    try:
        normalized_path = _normalize_path(filename, with_prefix=False)
    except ValueError as e:
        raise ManifestParseException(
            f'Error parsing the path "{filename}" defined in {attribute_path.path}: {e.args[0]}'
        ) from None
    if not allow_directories and filename.endswith("/"):
        raise ManifestParseException(
            f'The path "{filename}" in {attribute_path.path} ends with "/" implying it is a directory,'
            f" but this feature can only be used for files"
        )
    if normalized_path == ".":
        raise ManifestParseException(
            f'The path "{filename}" in {attribute_path.path} looks like the root directory,'
            f" but this feature does not allow the root directory here."
        )
    return normalized_path


def _parse_conffile_prior_version_and_owning_package(
    d: DpkgConffileManagementRuleBase,
    attribute_path: AttributePath,
) -> Tuple[Optional[str], Optional[str]]:
    prior_version = d.get("prior_to_version")
    owning_package = d.get("owning_package")

    if prior_version is not None and not PKGVERSION_REGEX.match(prior_version):
        p = attribute_path["prior_to_version"]
        raise ManifestParseException(
            f"The {MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION} parameter in {p.path} must be a"
            r" valid package version (i.e., match (?:\d+:)?\d[0-9A-Za-z.+:~]*(?:-[0-9A-Za-z.+:~]+)*)."
        )

    if owning_package is not None and not PKGNAME_REGEX.match(owning_package):
        p = attribute_path["owning_package"]
        raise ManifestParseException(
            f"The {MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE} parameter in {p.path} must be a valid"
            f" package name (i.e., match {PKGNAME_REGEX.pattern})."
        )

    return prior_version, owning_package


@dpkg_conffile_parser(_DPKG_RENAME_CONFFILE_PARSER)
def _dpkg_conffile_rename(
    _name: str,
    parsed_data: DpkgRenameConffileRule,
    path: AttributePath,
    _context: ParserContextData,
) -> DpkgMaintscriptHelperCommand:
    source_file = parsed_data["source"]
    target_file = parsed_data["target"]
    normalized_source = _parse_filename(
        source_file,
        path["source"],
        allow_directories=False,
    )
    path.path_hint = source_file

    normalized_target = _parse_filename(
        target_file,
        path["target"],
        allow_directories=False,
    )
    normalized_source = "/" + normalized_source
    normalized_target = "/" + normalized_target

    if normalized_source == normalized_target:
        raise ManifestParseException(
            f"Invalid rename defined in {path.path}: The source and target path are the same!"
        )

    version, owning_package = _parse_conffile_prior_version_and_owning_package(
        parsed_data, path
    )
    return DpkgMaintscriptHelperCommand.mv_conffile(
        path,
        normalized_source,
        normalized_target,
        version,
        owning_package,
    )


@dpkg_conffile_parser(_DPKG_REMOVE_CONFFILE_PARSER)
def _dpkg_conffile_remove(
    _name: str,
    parsed_data: DpkgRemoveConffileRule,
    path: AttributePath,
    _context: ParserContextData,
) -> DpkgMaintscriptHelperCommand:
    source_file = parsed_data["path"]
    normalized_source = _parse_filename(
        source_file,
        path["path"],
        allow_directories=False,
    )
    path.path_hint = source_file

    normalized_source = "/" + normalized_source

    version, owning_package = _parse_conffile_prior_version_and_owning_package(
        parsed_data, path
    )
    return DpkgMaintscriptHelperCommand.rm_conffile(
        path,
        normalized_source,
        version,
        owning_package,
    )


class HighLevelManifestParser(ParserContextData):
    def __init__(
        self,
        manifest_path: str,
        binary_packages: Mapping[str, BinaryPackage],
        substitution: Substitution,
        dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
        dpkg_arch_query_table: DpkgArchTable,
        build_env: DebOptionsAndProfiles,
        plugin_provided_feature_set: PluginProvidedFeatureSet,
        *,
        # Available for testing purposes only
        debian_dir: Union[str, VirtualPath] = "./debian",
    ):
        self.manifest_path = manifest_path
        self._binary_packages = binary_packages
        self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None
        self._substitution = substitution
        self._dpkg_architecture_variables = dpkg_architecture_variables
        self._dpkg_arch_query_table = dpkg_arch_query_table
        self._build_env = build_env
        self._substitution_stack = [self._substitution]
        self._package_state_stack = []
        self._plugin_provided_feature_set = plugin_provided_feature_set

        if isinstance(debian_dir, str):
            debian_dir = FSROOverlay("debian", debian_dir, None)

        self._debian_dir = debian_dir

        self._all_package_states = {
            n: PackageTransformationDefinition(
                binary_package=p,
                substitution=substitution.with_extra_substitutions(
                    **_per_package_subst_variables(p)
                ),
                is_auto_generated_package=False,
                maintscript_snippets=collections.defaultdict(
                    MaintscriptSnippetContainer
                ),
            )
            for n, p in binary_packages.items()
        }
        for n, p in binary_packages.items():
            dbgsym_name = f"{n}-dbgsym"
            if dbgsym_name in self._all_package_states:
                continue
            self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
                binary_package=p,
                substitution=substitution.with_extra_substitutions(
                    **_per_package_subst_variables(p, name=dbgsym_name)
                ),
                is_auto_generated_package=True,
                maintscript_snippets=collections.defaultdict(
                    MaintscriptSnippetContainer
                ),
            )

        self._install_rules: Optional[List[InstallRule]] = None
        self._ownership_caches_loaded = False
        self._used = False

    @property
    def binary_packages(self) -> Mapping[str, BinaryPackage]:
        return self._binary_packages

    @property
    def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
        return self._all_package_states

    def build_manifest(self) -> HighLevelManifest:
        if self._used:
            raise TypeError("build_manifest can only be called once!")
        self._used = True
        if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
            self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
        all_packager_provided_files = detect_all_packager_provided_files(
            self._plugin_provided_feature_set.packager_provided_files,
            self._debian_dir,
            self.binary_packages,
        )

        for package in self._package_states:
            with self.binary_package_context(package) as context:
                if not context.is_auto_generated_package:
                    ppf_result = all_packager_provided_files[package]
                    if ppf_result.auto_installable:
                        context.install_rules.append(
                            PPFInstallRule(
                                context.binary_package,
                                context.substitution,
                                ppf_result.auto_installable,
                            )
                        )
                    context.reserved_packager_provided_files.update(
                        ppf_result.reserved_only
                    )
                self._transform_dpkg_maintscript_helpers_to_snippets()

        return HighLevelManifest(
            self.manifest_path,
            self._mutable_yaml_manifest,
            self._install_rules,
            self.binary_packages,
            self.substitution,
            self._package_states,
            self._dpkg_architecture_variables,
            self._dpkg_arch_query_table,
            self._build_env,
            self._plugin_provided_feature_set,
        )

    @contextlib.contextmanager
    def binary_package_context(
        self, package_name: str
    ) -> Iterator[PackageTransformationDefinition]:
        if package_name not in self._package_states:
            self._error(
                f'The package "{package_name}" is not present in the debian/control file (could not find'
                f' "Package: {package_name}" in a binary paragraph) nor is it a -dbgsym package for one'
                " for a package in debian/control."
            )
        package_state = self._package_states[package_name]
        self._substitution_stack.append(package_state.substitution)
        self._package_state_stack.append(package_state)
        ss_len = len(self._substitution_stack)
        ps_len = len(self._package_state_stack)
        yield package_state
        if ss_len != len(self._substitution_stack) or ps_len != len(
            self._package_state_stack
        ):
            raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
        self._substitution_stack.pop()
        self._package_state_stack.pop()

    @property
    def substitution(self) -> Substitution:
        return self._substitution_stack[-1]

    @property
    def current_binary_package_state(self) -> PackageTransformationDefinition:
        if not self._package_state_stack:
            raise RuntimeError("Invalid state: Not in a binary package context")
        return self._package_state_stack[-1]

    def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
        package_state = self.current_binary_package_state
        for dmh in package_state.dpkg_maintscript_helper_snippets:
            snippet = MaintscriptSnippet(
                definition_source=dmh.definition_source,
                snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
            )
            for script in STD_CONTROL_SCRIPTS:
                package_state.maintscript_snippets[script].append(snippet)

    def normalize_path(
        self,
        path: str,
        definition_source: AttributePath,
        *,
        allow_root_dir_match: bool = False,
    ) -> ExactFileSystemPath:
        try:
            normalized = _normalize_path(path)
        except ValueError:
            self._error(
                f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
                ' package and not use any ".." or "." segments.'
            )
        if normalized == "." and not allow_root_dir_match:
            self._error(
                "Manifests must not change the root directory of the deb file.  Please correct"
                f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
            )
        return ExactFileSystemPath(
            self.substitution.substitute(normalized, definition_source.path)
        )

    def parse_path_or_glob(
        self,
        path_or_glob: str,
        definition_source: AttributePath,
    ) -> MatchRule:
        match_rule = MatchRule.from_path_or_glob(
            path_or_glob, definition_source.path, substitution=self.substitution
        )
        # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
        # so there is no need to check for an exact match on "." like in normalize_path.
        if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
            self._error(
                f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
                f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
                f' something that matches "less" than everything.'
            )
        return match_rule

    def parse_manifest(self) -> HighLevelManifest:
        raise NotImplementedError


class YAMLManifestParser(HighLevelManifestParser):
    def _optional_key(
        self,
        d,
        key,
        attribute_parent_path: AttributePath,
        expected_type=None,
        default_value=None,
    ):
        v = d.get(key)
        if v is None:
            _detect_possible_typo(d, key, attribute_parent_path, False)
            return default_value
        if expected_type is not None:
            return self._ensure_value_is_type(
                v, expected_type, key, attribute_parent_path
            )
        return v

    def _required_key(
        self,
        d,
        key,
        attribute_parent_path: AttributePath,
        expected_type=None,
        extra: Optional[Union[str, Callable[[], str]]] = None,
    ):
        v = d.get(key)
        if v is None:
            _detect_possible_typo(d, key, attribute_parent_path, True)
            if extra is not None:
                msg = extra if isinstance(extra, str) else extra()
                extra_info = " " + msg
            else:
                extra_info = ""
            self._error(
                f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
                f"{extra_info}"
            )

        if expected_type is not None:
            return self._ensure_value_is_type(
                v, expected_type, key, attribute_parent_path
            )
        return v

    def _ensure_value_is_type(
        self,
        v,
        t,
        key: Union[str, int, AttributePath],
        attribute_parent_path: Optional[AttributePath],
    ):
        if v is None:
            return None
        if not isinstance(v, t):
            if isinstance(t, tuple):
                t_msg = "one of: " + ", ".join(x.__name__ for x in t)
            else:
                t_msg = f"a {t.__name__}"
            key_path = (
                key.path
                if isinstance(key, AttributePath)
                else attribute_parent_path[key].path
            )
            self._error(
                f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
            )
        return v

    def _unpack_single_key_dict(self, value, definition_source: AttributePath) -> str:
        self._ensure_value_is_type(value, dict, definition_source, None)
        if len(value) != 1:
            self._error(
                f'The mapping "{definition_source}" had two keys, but it should only have one top level key.'
                " Maybe you are missing a list marker behind the second key or some indentation.  The keys"
                f' are: {", ".join(value)}'
            )
        return next(iter(value))

    def _parse_yaml_condition_not(
        self,
        d,
        key: str,
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        not_def = self._required_key(d, key, orig_definition_source, expected_type=dict)
        condition = self.parse_manifest_condition(not_def, orig_definition_source[key])
        return condition.negated()

    def _parse_yaml_condition_group(
        self,
        d,
        key: str,
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        group_def = self._required_key(
            d, key, orig_definition_source, expected_type=list
        )
        definition_source = orig_definition_source[key]
        conditions = [
            self.parse_manifest_condition(cd, definition_source[i])
            for i, cd in enumerate(group_def)
        ]
        if len(conditions) < 2:
            self._error(
                f"Invalid condition at {definition_source.path}, the {key} requires at least 2 conditions."
            )

        if key == "any-of":
            return ManifestCondition.any_of(conditions)
        assert key == "all-of"
        return ManifestCondition.all_of(conditions)

    def _parse_yaml_condition_arch_matches(
        self,
        d,
        key: str,
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        arch_matches_raw = self._required_key(
            d,
            key,
            orig_definition_source,
            expected_type=str,
        )
        package_state = self.current_binary_package_state
        arch_matches_as_list = arch_matches_raw.split()
        dpkg_arch_query = self._dpkg_arch_query_table

        if not arch_matches_as_list:
            self._error(
                f"The condition at {orig_definition_source.path} must not be empty"
            )
        if package_state.binary_package.is_arch_all:
            if arch_matches_as_list[0].startswith("["):
                arch_matches_as_list[0] = arch_matches_as_list[0][1:]
            if arch_matches_as_list[-1].endswith("]"):
                arch_matches_as_list[-1] = arch_matches_as_list[-1][:-1]
            result = dpkg_arch_query.architecture_is_concerned(
                "all", arch_matches_as_list
            )
            self._error(
                f"The package architecture restriction at {orig_definition_source.path} is applied to the"
                f' "Architecture: all" package {package_state.binary_package.name}, which does not make sense'
                f" as the condition will always resolves to `{str(result).lower()}`."
            )
        if arch_matches_as_list[0].startswith("[") or arch_matches_as_list[-1].endswith(
            "]"
        ):
            self._error(
                f"The architecture match at {orig_definition_source} must be defined without enclosing it with "
                '"[" or/and "]" brackets'
            )
        # Can we check arch list for typos.  If we do, it must be tight in how close matches it does.
        # Consider "arm" vs. "armel" (edit distance 2, but both are valid).  Likewise, names often
        # include a bit indicator "foo", "foo32", "foo64" - all of these have an edit distance of 2
        # of each other.
        return ArchMatchManifestCondition(arch_matches_as_list)

    def _parse_yaml_condition_build_profiles_matches(
        self,
        d,
        key: str,
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        build_profile_spec = self._required_key(
            d,
            key,
            orig_definition_source,
            expected_type=str,
        )
        build_profile_spec = build_profile_spec.strip()
        if not build_profile_spec:
            self._error(
                f"The condition at {orig_definition_source.path} must not be empty"
            )
        try:
            active_profiles_match(build_profile_spec, frozenset())
        except ValueError as e:
            self._error(
                f"Could not parse the build specification at {orig_definition_source.path}: {e.args[0]}"
            )
        return BuildProfileMatch(build_profile_spec)

    def parse_manifest_condition(
        self,
        d: Union[str, Mapping[str, Any]],
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        supported_mapping_conditions: Dict[
            str, Callable[[Mapping, str, AttributePath], ManifestCondition]
        ] = {
            "not": self._parse_yaml_condition_not,
            MK_CONDITION_ARCH_MATCHES: self._parse_yaml_condition_arch_matches,
            MK_CONDITION_BUILD_PROFILES_MATCHES: self._parse_yaml_condition_build_profiles_matches,
            "any-of": self._parse_yaml_condition_group,
            "all-of": self._parse_yaml_condition_group,
        }
        supported_str_conditions: Dict[str, ManifestCondition] = {
            "cross-compiling": ManifestCondition.is_cross_building(),
            "can-execute-compiled-binaries": ManifestCondition.can_execute_compiled_binaries(),
            "run-build-time-tests": ManifestCondition.run_build_time_tests(),
        }
        condition = None
        if isinstance(d, str):
            condition_key = d
            condition = supported_str_conditions.get(condition_key)
        else:
            condition_key = self._unpack_single_key_dict(d, orig_definition_source)

            handler = supported_mapping_conditions.get(condition_key)
            if handler is not None:
                condition = handler(d, condition_key, orig_definition_source)

        if condition is None:
            if condition_key in supported_mapping_conditions:
                assert not isinstance(d, dict)
                self._error(
                    f"Invalid definition at {orig_definition_source.path}.  The condition {condition_key} must be a "
                    "mapping."
                )
            elif condition_key in supported_str_conditions:
                self._error(
                    f"Invalid definition at {orig_definition_source.path}.  The condition {condition_key} must be a "
                    "string."
                )
            dict_options = (f"{k} (mapping)" for k in supported_mapping_conditions)
            string_options = (f"{k} (string)" for k in supported_str_conditions)
            valid_options = ", ".join(
                sorted(itertools.chain(dict_options, string_options))
            )
            self._error(
                f'Unknown or unsupported condition "{condition_key}" at {orig_definition_source.path}.'
                f" Valid conditions are: {valid_options}"
            )

        return condition

    def _parse_yaml_installations(
        self,
        d: ManifestRootRule,
        orig_definition_source: AttributePath,
    ) -> None:
        raw_installations = d.get("installations")

        if raw_installations is None and MK_INSTALLATIONS not in d:
            # Explicitly empty vs. implicitly empty matters in corner cases, so preserve this
            # fact.
            return

        self._install_rules: List[InstallRule] = []
        parent_path = orig_definition_source[MK_INSTALLATIONS]

        for no, raw_installation in enumerate(raw_installations):
            definition_source = parent_path[no]
            installation_key = self._unpack_single_key_dict(
                raw_installation,
                definition_source,
            )
            handler = _INSTALL_RULE_PARSERS.get(installation_key)
            if handler is not None:
                entry_path = definition_source[installation_key]
                parsed_data = handler.parser.parse_input(
                    raw_installation[installation_key], entry_path, parser_context=self
                )
                installation = handler.handler(
                    installation_key, parsed_data, entry_path, self
                )
                self._install_rules.append(installation)
            else:
                self._error(
                    f'Unknown or unsupported install rule "{installation_key}" at {definition_source.path}.'
                    " Installations must have one of the following keys:"
                    f' {", ".join(sorted(_INSTALL_RULE_PARSERS))}'
                )

    def _parse_yaml_transformations(
        self,
        d: BinaryPackageRule,
        transformations: List[TransformationRule],
        orig_definition_source: AttributePath,
    ) -> None:
        raw_transformations = d.get("transformations", [])
        parent_path = orig_definition_source["transformations"]

        for no, raw_transformation in enumerate(raw_transformations):
            definition_source = parent_path[no]
            transformation_key = self._unpack_single_key_dict(
                raw_transformation,
                definition_source,
            )
            handler = _TRANSFORMATION_RULE_PARSERS.get(transformation_key)
            if handler is not None:
                entry_path = definition_source[transformation_key]
                parsed_data = handler.parser.parse_input(
                    raw_transformation[transformation_key],
                    entry_path,
                    parser_context=self,
                )
                transformation = handler.handler(
                    transformation_key, parsed_data, entry_path, self
                )
                transformations.append(transformation)
            else:
                self._error(
                    f'Unknown or unsupported transformation "{transformation_key}" at {definition_source}.'
                    " Transformations must have one of the following keys:"
                    f' {", ".join(sorted(_TRANSFORMATION_RULE_PARSERS))}'
                )

    def _parse_yaml_conffile_management(
        self,
        d: BinaryPackageRule,
        definition_source: AttributePath,
    ) -> None:
        entries = d.get("conffile_management", [])
        package_state = self.current_binary_package_state

        parent_path = definition_source[MK_CONFFILE_MANAGEMENT]

        for no, entry in enumerate(entries):
            path_definition_source = parent_path[no]
            key = self._unpack_single_key_dict(entry, path_definition_source)
            handler = _DPKG_CONFFILE_MGNT_PARSERS.get(key)
            if handler is not None:
                entry_path = path_definition_source[key]
                parsed_data = handler.parser.parse_input(
                    entry[key], entry_path, parser_context=self
                )
                dpkg_script_snippet = handler.handler(
                    key, parsed_data, entry_path, self
                )
                package_state.dpkg_maintscript_helper_snippets.append(
                    dpkg_script_snippet
                )
            else:
                self._error(
                    f'Unknown or unsupported conffile management "{key}" at {path_definition_source.path}.'
                    " Conffile management must have one of the following keys:"
                    f' {", ".join(sorted(entry))}'
                )

    def from_yaml_dict(self, yaml_data):
        attribute_path = AttributePath.root_path()
        parsed_data = _MANIFEST_ROOT_PARSER.parse_input(
            yaml_data, attribute_path, parser_context=self
        )
        manifest_version = parsed_data["manifest_version"]
        if manifest_version not in SUPPORTED_MANIFEST_VERSIONS:
            self._error(
                "Unsupported manifest-version.  This implementation supports the following versions:"
                f' {", ".join(repr(v) for v in SUPPORTED_MANIFEST_VERSIONS)}"'
            )

        packages_dict = parsed_data.get("packages", {})
        self._parse_yaml_installations(parsed_data, attribute_path)
        packages_parent_path = attribute_path["packages"]
        for k, v in packages_dict.items():
            definition_source = packages_parent_path[k]
            with self.binary_package_context(k) as package_state:
                if package_state.is_auto_generated_package:
                    # Maybe lift (part) of this restriction.
                    self._error(
                        f'Cannot define rules for package "{k}" (at {definition_source.path}). It is an'
                        " auto-generated package."
                    )
                parsed = _BINARY_PACKAGE_RULE_PARSER.parse_input(
                    v, definition_source, parser_context=self
                )

                binary_version = parsed.get("binary_version")
                if binary_version is not None:
                    package_state.binary_version = (
                        package_state.substitution.substitute(
                            binary_version,
                            definition_source["binary-version"],
                        )
                    )
                self._parse_yaml_transformations(
                    parsed,
                    package_state.transformations,
                    definition_source,
                )
                self._parse_yaml_conffile_management(parsed, definition_source)
        return self.build_manifest()

    def parse_manifest(self, *, fd: Union[IO[bytes], str] = None) -> HighLevelManifest:
        if fd is None:
            with open(self.manifest_path) as fd:
                data = MANIFEST_YAML.load(fd)
                self._mutable_yaml_manifest = MutableYAMLManifest(data)
                return self.from_yaml_dict(data)
        else:
            data = MANIFEST_YAML.load(fd)
            self._mutable_yaml_manifest = MutableYAMLManifest(data)
            return self.from_yaml_dict(data)
