Coverage for debputy/highlevel_manifest_parser.py: 0%
602 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1import collections
2import contextlib
3import itertools
4import operator
5import os
6from typing import Iterable, Tuple, Optional, Dict, TypeVar, Callable, List, Any, Union, Mapping, \
7 Generator
9from debian.debian_support import DpkgArchTable
11from debputy.builtin_manifest_rules import builtin_manifest_rules, DEFAULT_PATH_INFO
12from debputy.highlevel_manifest import HighLevelManifest, PackageTransformationDefinition, MutableYAMLManifest, \
13 MANIFEST_YAML
14from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand, MaintscriptSnippet, STD_CONTROL_SCRIPTS
15from debputy.manifest_path_rules import ManifestPathRule
16from debputy.packages import BinaryPackage
17from debputy.path_matcher import MatchRuleType, ExactFileSystemPath, MatchRule
18from debputy.substitution import Substitution
19from debputy.transformation_rules import TransformationRule, ExclusionTransformationRule, \
20 CreateDirectoryTransformationRule, CreateVirtualPathTransformationRule, MoveTransformationRule
21from debputy.util import _normalize_path, _error, PKGVERSION_REGEX, PKGNAME_REGEX, escape_shell, active_profiles_match
22from ._deb_options_profiles import DebOptionsAndProfiles
23from ._manifest_constants import *
24from .architecture_support import DpkgArchitectureBuildProcessValuesTable
25from .manifest_conditions import ManifestCondition, ArchMatchManifestCondition, BuildProfileMatch
28class OwnershipDefinition:
29 __slots__ = ('entity_name', 'entity_id')
31 def __init__(self, entity_name, entity_id):
32 self.entity_name = entity_name
33 self.entity_id = entity_id
36ROOT_DEFINITION = OwnershipDefinition('root', 0)
38BAD_OWNER_NAMES = {
39 '_apt', # All things owned by _apt are generated by apt after installation
40 'nogroup', # It is not supposed to own anything as it is an entity used for dropping permissions
41 'nobody', # It is not supposed to own anything as it is an entity used for dropping permissions
42}
43BAD_OWNER_IDS = {
44 65534, # ID of nobody / nogroup
45}
47OSK = TypeVar('OSK')
48OSV = TypeVar('OSV')
51def _full_key_name(key, key_prefix):
52 return key if not key_prefix else key_prefix + "." + key
55def _parse_keyword(keyword: str) -> Tuple[str, str]:
56 v = keyword.split('=', 1)
57 return v[0], v[1]
60def _read_ownership_def_from_base_password_template(template_file: str) -> Iterable[OwnershipDefinition]:
61 with open(template_file) as fd:
62 for line in fd:
63 entity_name, _star, entity_id, _remainder = line.split(":", 3)
64 yield OwnershipDefinition(entity_name, int(entity_id))
67def _parse_ownership(v: str) -> Tuple[Optional[str], Optional[int]]:
68 if isinstance(v, str) and ':' in v:
69 if v == ':':
70 raise ValueError('Ownership is redundant if it is ":" (blank name and blank id)')
71 entity_name, entity_id = v.split(':')
72 if entity_name == '':
73 entity_name = None
74 if entity_id != '':
75 entity_id = int(entity_id)
76 else:
77 entity_id = None
78 return entity_name, entity_id
80 if isinstance(v, int):
81 return None, v
82 return v, None
85def _resolve_entity(key: OSK, key_type: str, cache_lookup: Callable[[OSK], OSV], expected_result: Optional[OSV] = None):
86 try:
87 resolved_entity_value = cache_lookup(key)
88 except KeyError:
89 raise ValueError(
90 f"Refusing to use {key} as {key_type}: It is not known to be a static {key_type} from base-passwd")
92 if expected_result is not None and expected_result != resolved_entity_value:
93 raise ValueError(
94 f"Bad {key_type} declaration: Looking up {key} resolves to {resolved_entity_value} according to"
95 f" base-passwd, but the packager declared to should have been {expected_result}")
96 return resolved_entity_value
99class HighLevelManifestParser:
101 def __init__(self,
102 manifest_path: str,
103 binary_packages: Mapping[str, BinaryPackage],
104 substitution: Substitution,
105 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
106 dpkg_arch_query_table: DpkgArchTable,
107 build_env: DebOptionsAndProfiles,
108 ):
109 self.manifest_path = manifest_path
110 self.binary_packages = binary_packages
111 self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None
112 self._substitution = substitution
113 self._dpkg_architecture_variables = dpkg_architecture_variables
114 self._dpkg_arch_query_table = dpkg_arch_query_table
115 self._build_env = build_env
116 self._substitution_stack = [self._substitution]
117 self._package_state_stack = []
119 self._package_states = {
120 n: PackageTransformationDefinition(
121 binary_package=p,
122 binary_version=None,
123 ensure_directories={},
124 create_virtual_paths={},
125 path_specs={},
126 substitution=substitution.with_extra_substitutions(PACKAGE=n),
127 dpkg_maintscript_helper_snippets=[],
128 maintscript_snippets=collections.defaultdict(list),
129 transformations=[],
130 )
131 for n, p in binary_packages.items()
132 }
134 self._owner_cache = {ROOT_DEFINITION.entity_name: ROOT_DEFINITION}
135 self._group_cache = {ROOT_DEFINITION.entity_name: ROOT_DEFINITION}
136 self._uid_cache = {v.entity_id: v for v in self._owner_cache.values()}
137 self._gid_cache = {v.entity_id: v for v in self._group_cache.values()}
138 self._ownership_caches_loaded = False
139 self._used = False
141 def build_manifest(self) -> HighLevelManifest:
142 if self._used:
143 raise TypeError("build_manifest can only be called once!")
144 self._used = True
145 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
146 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
147 for package in self.binary_packages:
148 with self.binary_package_context(package) as context:
149 for manifest_info in builtin_manifest_rules(context.substitution):
150 self._add_entry(manifest_info)
151 if context.ensure_directories:
152 context.transformations.append(
153 CreateDirectoryTransformationRule(context.ensure_directories.values())
154 )
155 if context.create_virtual_paths:
156 context.transformations.append(CreateVirtualPathTransformationRule(
157 context.create_virtual_paths.values()
158 ))
159 self._transform_dpkg_maintscript_helpers_to_snippets()
160 return HighLevelManifest(
161 self.manifest_path,
162 self._mutable_yaml_manifest,
163 self.binary_packages,
164 self.substitution,
165 self._package_states,
166 self._dpkg_architecture_variables,
167 self._dpkg_arch_query_table,
168 self._build_env,
169 )
171 @contextlib.contextmanager
172 def binary_package_context(self, package_name: str) -> Generator[PackageTransformationDefinition, Any, Any]:
173 if package_name not in self.binary_packages:
174 _error(f'The package "{package_name}" is not present in the debian/control file (could not find'
175 f' "Package: {package_name}" in a binary paragraph)')
176 package_state = self._package_states[package_name]
177 self._substitution_stack.append(package_state.substitution)
178 self._package_state_stack.append(package_state)
179 ss_len = len(self._substitution_stack)
180 ps_len = len(self._package_state_stack)
181 yield package_state
182 if ss_len != len(self._substitution_stack) or ps_len != len(self._package_state_stack):
183 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
184 self._substitution_stack.pop()
185 self._package_state_stack.pop()
187 @property
188 def substitution(self) -> Substitution:
189 return self._substitution_stack[-1]
191 @property
192 def current_binary_package_state(self) -> PackageTransformationDefinition:
193 if not self._package_state_stack:
194 raise RuntimeError("Invalid state: Not in a binary package context")
195 return self._package_state_stack[-1]
197 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
198 package_state = self.current_binary_package_state
199 for dmh in package_state.dpkg_maintscript_helper_snippets:
200 snippet = MaintscriptSnippet(
201 definition_source=dmh.definition_source,
202 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
203 )
204 for script in STD_CONTROL_SCRIPTS:
205 package_state.maintscript_snippets[script].append(snippet)
207 def normalize_path(self, path: str, definition_source: str) -> ExactFileSystemPath:
208 normalized = _normalize_path(path)
209 if normalized == '.':
210 _error('Manifests must not change the root directory of the deb file. Please correct'
211 f' "{definition_source}" (path: "{path}) in {self.manifest_path}')
212 return ExactFileSystemPath(self.substitution.substitute(normalized, definition_source))
214 def parse_path_or_glob(self, path_or_glob: str, definition_source: str) -> MatchRule:
215 match_rule = MatchRule.from_path_or_glob(path_or_glob, definition_source, substitution=self.substitution)
216 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
217 # so there is no need to check for an exact match on "." like in normalize_path.
218 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
219 _error(f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
220 f' Please correct "{definition_source}" (path: "{path_or_glob}) in {self.manifest_path} to'
221 f' something that matches "less" than everything.')
222 return match_rule
224 def parse_manifest(self) -> HighLevelManifest:
225 raise NotImplementedError
227 def _load_ownership_cache(self) -> None:
228 for owner_def in _read_ownership_def_from_base_password_template('/usr/share/base-passwd/passwd.master'):
229 if owner_def.entity_id == 0:
230 # Leave root as it is
231 continue
233 # Could happen if base-passwd template has two users with the same ID. We assume this will not occur.
234 assert owner_def.entity_name not in self._owner_cache
235 assert owner_def.entity_id not in self._uid_cache
236 self._owner_cache[owner_def.entity_name] = owner_def
237 self._uid_cache[owner_def.entity_id] = owner_def
238 for group_def in _read_ownership_def_from_base_password_template('/usr/share/base-passwd/group.master'):
239 if group_def.entity_id == 0:
240 # Leave root as it is
241 continue
243 # Could happen if base-passwd template has two groups with the same ID. We assume this will not occur.
244 assert group_def.entity_name not in self._group_cache
245 assert group_def.entity_id not in self._gid_cache
246 self._group_cache[group_def.entity_name] = group_def
247 self._gid_cache[group_def.entity_id] = group_def
248 self._ownership_caches_loaded = True
250 def parse_and_resolve_owner(self, declaration: Optional[str]):
251 return self.parse_and_resolve_ownership(
252 declaration,
253 'owner',
254 self._owner_cache,
255 self._uid_cache
256 )
258 def parse_and_resolve_group(self, declaration: Optional[str]):
259 return self.parse_and_resolve_ownership(
260 declaration,
261 'group',
262 self._group_cache,
263 self._gid_cache
264 )
266 def parse_and_resolve_ownership(self,
267 declaration: Optional[str],
268 ownership_type: str,
269 name_cache: Dict[str, OwnershipDefinition],
270 id_cache: Dict[int, OwnershipDefinition],
271 ):
272 if declaration is None:
273 return None, None
274 entity_name, entity_id = _parse_ownership(declaration)
275 return self.resolve_ownership(entity_name, entity_id, ownership_type, name_cache, id_cache)
277 def resolve_ownership(self,
278 entity_name: Optional[str],
279 entity_id: Optional[int],
280 ownership_type: str,
281 name_cache: Dict[str, OwnershipDefinition],
282 id_cache: Dict[int, OwnershipDefinition],
283 ) -> Tuple[str, int]:
284 if not self._ownership_caches_loaded:
285 if (entity_name is not None and entity_name not in name_cache) or \
286 (entity_id is not None and entity_id not in id_cache):
287 self._load_ownership_cache()
288 assert self._ownership_caches_loaded
289 resolved_name = entity_name
290 resolved_id = entity_id
291 if entity_id is not None:
292 resolved_name = _resolve_entity(entity_id,
293 ownership_type,
294 lambda x: id_cache[x].entity_name,
295 expected_result=entity_name)
296 if entity_name is not None:
297 resolved_id = _resolve_entity(entity_name,
298 ownership_type,
299 lambda x: name_cache[x].entity_id,
300 expected_result=entity_id)
301 if resolved_name in BAD_OWNER_NAMES or resolved_id in BAD_OWNER_IDS:
302 raise ValueError(f'Refusing to use {resolved_name}:{resolved_id} as {ownership_type}.'
303 ' No path should have this entity as owner/group as it is unsafe.')
304 if resolved_name is None or resolved_id is None:
305 raise AssertionError(f"Internal error: The base-passwd template could not fully resolve {ownership_type}"
306 f" with input name {entity_name} and/or id {entity_id}."
307 " The full ownership information is necessary to generate a correct data.tar file!"
308 " Please file a bug for this tool for now.")
309 # The asserts here are mostly typing hints
310 assert resolved_name is not None
311 assert resolved_id is not None
312 return resolved_name, resolved_id
314 def _check_existing(self,
315 match_rule: MatchRule,
316 path_info: ManifestPathRule,
317 ) -> bool:
318 package_state = self.current_binary_package_state
319 if match_rule in package_state.path_specs:
320 other_path_spec = package_state.path_specs[match_rule]
321 if path_info.is_builtin_definition and not other_path_spec.is_builtin_definition:
322 return False
323 _error(f'The path "{match_rule.describe_match_short()}" has conflicting definitions (or is defined twice)'
324 f' in manifest {self.manifest_path}. The two definitions are:\n'
325 f' * {other_path_spec.definition_source}\n'
326 f' * {path_info.definition_source}\n'
327 'Please consolidate them into one definition.')
328 if path_info.is_builtin_definition and match_rule.is_path_type_part_of_match:
329 # if the user provided a rule that overlaps with a built-in rule, we also discard the built-in rule, if
330 # has a path type restriction while the user provided does not. Notably, users are currently unable to
331 # provide a path type restricted rules.
332 #
333 # This is the logic that ensures that the user can write "remove *.la" and not be steam rolled by the
334 # built-in "normalize permissions for *.la files" rule.
335 shadowing_match_rule = match_rule.match_rule_without_path_type_match()
336 return shadowing_match_rule not in package_state.path_specs
337 return True
339 def _add_entry(self, path_info: ManifestPathRule) -> None:
340 package_state = self.current_binary_package_state
341 match_rule = path_info.match_rule
342 if not self._check_existing(match_rule, path_info):
343 return
344 package_state.path_specs[match_rule] = path_info
345 if path_info.rule_type.creates_path:
346 assert isinstance(match_rule, ExactFileSystemPath)
347 assert path_info.member_path not in package_state.create_virtual_paths
348 package_state.create_virtual_paths[path_info.member_path] = path_info
349 current_path_info = path_info
350 parent_dir = ExactFileSystemPath(os.path.dirname(match_rule.path))
351 assert parent_dir.path.startswith("./")
352 while True:
353 parent_path_info = package_state.path_specs.get(parent_dir)
354 existed = False
355 if parent_path_info is None:
356 parent_path_info = ManifestPathRule.directory(
357 path_info.definition_source,
358 parent_dir,
359 triggered_by=path_info,
360 )
361 package_state.path_specs[parent_dir] = parent_path_info
362 else:
363 existed = True
364 parent_path_info.add_virtual_child(current_path_info)
365 if existed or parent_dir.path == '.':
366 break
367 current_path_info = parent_path_info
368 parent_dir = ExactFileSystemPath(os.path.dirname(parent_dir.path))
371class YAMLManifestParser(HighLevelManifestParser):
373 def _optional_key(self, d, key, key_prefix, expected_type=None, default_value=None):
374 v = d.get(key)
375 if v is None:
376 return default_value
377 if expected_type is not None:
378 return self._ensure_value_is_type(v, expected_type, key, key_prefix)
379 return v
381 def _required_key(self, d, key, key_prefix: str, expected_type=None, extra=None):
382 v = d.get(key)
383 if v is None:
384 key_path = _full_key_name(key, key_prefix)
385 extra_info = " " + extra() if extra is not None else ''
386 _error(f'Missing required key {key_path} in manifest "{self.manifest_path}.{extra_info}"')
388 if expected_type is not None:
389 return self._ensure_value_is_type(v, expected_type, key, key_prefix)
390 return v
392 def _ensure_value_is_type(self, v, t, key: str, key_prefix: str):
393 if v is None:
394 return None
395 if not isinstance(v, t):
396 if isinstance(t, tuple):
397 t_msg = "one of: " + ', '.join(x.__name__ for x in t)
398 else:
399 t_msg = f'of type {t.__name__}'
400 _error(f"The key {_full_key_name(key, key_prefix)} (manifest: {self.manifest_path}) must be {t_msg}")
401 return v
403 def _parse_ownership_and_mode(self, d: Dict[str, Any], definition_source) -> Dict[str, Union[int, str]]:
404 mode = self._optional_key(d, 'mode', definition_source, expected_type=str)
405 try:
406 mode = int(mode, base=8) if mode is not None else None
407 except ValueError:
408 key_path = _full_key_name('mode', definition_source)
409 _error(f'Unable to parse {key_path} as an octal number (manifest: {self.manifest_path})')
410 declared_owner = self._optional_key(d, 'owner', definition_source,
411 expected_type=(str, int),
412 default_value='root:0')
413 declared_group = self._optional_key(d, 'group', definition_source,
414 expected_type=(str, int),
415 default_value='root:0')
416 try:
417 owner, uid = self.parse_and_resolve_owner(declared_owner)
418 except ValueError as e:
419 _error(f'Invalid ownership information in {_full_key_name("owner", definition_source)}'
420 f' (manifest: {self.manifest_path}): {e.args[0]}')
421 try:
422 group, gid = self.parse_and_resolve_group(declared_group)
423 except ValueError as e:
424 _error(f'Invalid ownership in {_full_key_name("group", definition_source)}'
425 f' (manifest: {self.manifest_path}): {e.args[0]}')
426 return {
427 "mode": mode,
428 "dirmode": mode,
429 "special_mode": mode,
430 "uid": uid,
431 "uname": owner,
432 "gid": gid,
433 "gname": group,
434 }
436 def _parse_yaml_path_metadata(self, d, orig_definition_source: str) -> None:
437 raw_path_specs = self._optional_key(d,
438 'path-metadata',
439 orig_definition_source,
440 expected_type=dict,
441 default_value={},
442 )
443 for path_spec, path_metadata in raw_path_specs.items():
444 definition_source = f'{orig_definition_source}.path-metadata'
445 # Ensure path_spec is a str for the error m
446 self._ensure_value_is_type(path_spec, str, f'{path_spec}', definition_source)
447 if path_metadata is None:
448 path_metadata = {}
449 else:
450 self._ensure_value_is_type(path_metadata, dict, path_spec, definition_source)
451 definition_source = f'{definition_source}."{path_spec}"'
453 ownership_and_mode = self._parse_ownership_and_mode(path_metadata, definition_source)
454 condition = self._parse_yaml_when_if_present(path_metadata, definition_source)
456 path_info = ManifestPathRule.ensure_metadata(
457 definition_source,
458 self.parse_path_or_glob(path_spec, definition_source),
459 **ownership_and_mode,
460 condition=condition,
461 )
462 self._add_entry(path_info)
464 def _parse_yaml_create_symlinks(self, d, orig_definition_source: str) -> None:
465 raw_path_specs = self._optional_key(d,
466 MK_CREATE_SYMLINKS,
467 orig_definition_source,
468 expected_type=list,
469 default_value=[],
470 )
471 package_state = self.current_binary_package_state
472 for no, symlink_spec in enumerate(raw_path_specs):
473 definition_source = f'{orig_definition_source}.{MK_CREATE_SYMLINKS}[{no}]'
474 self._ensure_value_is_type(symlink_spec, dict, definition_source, '')
475 if '*' in symlink_spec or '?' in symlink_spec:
476 _error(f'Globs are currently not supported {definition_source}')
477 link_path = self._optional_key(symlink_spec,
478 MK_CREATE_SYMLINKS_LINK_PATH,
479 definition_source,
480 expected_type=str,)
481 link_target = self._optional_key(symlink_spec,
482 MK_CREATE_SYMLINKS_LINK_TARGET,
483 definition_source,
484 expected_type=str,
485 )
486 condition = self._parse_yaml_when_if_present(symlink_spec, definition_source)
487 # Include the name in the source as it is easier for people to find than counting their way through
488 # the list.
489 definition_source = f'{definition_source} <{link_path}>'
490 normalized_link_path = self.normalize_path(link_path, definition_source)
491 path_info = ManifestPathRule.symlink(definition_source,
492 normalized_link_path,
493 link_target=self.substitution.substitute(link_target,
494 definition_source),
495 condition=condition,
496 )
497 directory = os.path.dirname(path_info.member_path)
498 if directory not in package_state.ensure_directories:
499 package_state.ensure_directories[directory] = ManifestPathRule.directory(
500 definition_source,
501 ExactFileSystemPath(directory),
502 triggered_by=path_info,
503 )
504 self._add_entry(path_info)
506 def _unpack_single_key_dict(self, value, definition_source: str) -> str:
507 self._ensure_value_is_type(value, dict, definition_source, '')
508 if len(value) != 1:
509 _error(f'The mapping "{definition_source}" had two keys, but it should only have one top level key.'
510 ' Maybe you are missing a list marker behind the second key or some indentation. The keys'
511 f' are: {", ".join(value)}')
512 return next(iter(value))
514 def _parse_yaml_directories(self, d, orig_definition_source: str) -> None:
515 raw_path_specs = self._optional_key(d,
516 'directories',
517 orig_definition_source,
518 expected_type=list,
519 default_value=[],
520 )
521 directories = []
522 package_state = self.current_binary_package_state
523 for no, dir_def in enumerate(raw_path_specs):
524 definition_source = f'{orig_definition_source}.directories[{no}]'
525 self._ensure_value_is_type(dir_def, (str, dict), definition_source, '')
526 if isinstance(dir_def, str):
527 dir_path = dir_def
528 dir_data = {}
529 else:
530 assert isinstance(dir_def, dict)
531 dir_data = dir_def
532 dir_path = self._required_key(dir_def, 'path', definition_source, expected_type=str)
534 ownership_and_mode = self._parse_ownership_and_mode(dir_data, definition_source)
535 condition = self._parse_yaml_when_if_present(dir_data, definition_source)
537 # Include the name in the source as it is easier for people to find than counting their way through
538 # the list.
539 definition_source = f'{definition_source} <{dir_path}>'
540 normalized_dir_path = self.normalize_path(dir_path, definition_source)
541 path_info = ManifestPathRule.directory(definition_source,
542 normalized_dir_path,
543 **ownership_and_mode,
544 condition=condition,
545 )
546 directories.append(path_info)
548 # Insert them in sort order as that will ensure the implicit path match the "shortest" rule rather than the
549 # "first" rule. It also saves us form having to do "conflict" resolution between implicit and explicit
550 # directory rules.
551 for path_info in sorted(directories, key=operator.attrgetter('member_path')):
552 self._add_entry(path_info)
553 assert path_info.member_path not in package_state.ensure_directories
554 package_state.ensure_directories[path_info.member_path] = path_info
556 def _parse_transformation_exclusion(self, d, key: str, definition_source) -> TransformationRule:
557 exclusion = self._required_key(d, key, definition_source, expected_type=str)
558 definition_source += f'.{key} <{exclusion}>'
559 return ExclusionTransformationRule(
560 self.parse_path_or_glob(exclusion, definition_source),
561 definition_source,
562 )
564 def _parse_transformation_move(self, d, key: str, definition_source: str) -> TransformationRule:
565 move_def = self._required_key(d, key, definition_source, expected_type=dict)
566 definition_source += f'.{key}'
567 source = self._required_key(move_def, 'source', definition_source, expected_type=str)
568 target = self._required_key(move_def, 'target', definition_source, expected_type=str)
569 condition = self._parse_yaml_when_if_present(d, definition_source)
570 definition_source += f' <{source}>'
571 source_match = self.parse_path_or_glob(source, definition_source)
572 target_path = self.normalize_path(target, definition_source).path
573 if isinstance(source_match, ExactFileSystemPath) and source_match.path == target_path:
574 _error(f'The transformation rule {definition_source} requests a move of {source_match} to {target_path},'
575 f' which is the same path')
576 return MoveTransformationRule(
577 source_match,
578 target_path,
579 target.endswith('/'),
580 definition_source,
581 condition,
582 )
584 def _parse_yaml_condition_not(self, d, key: str, orig_definition_source: str) -> ManifestCondition:
585 not_def = self._required_key(d, key, orig_definition_source, expected_type=dict)
586 condition = self._parse_yaml_condition(not_def, f'{orig_definition_source}.{key}')
587 return condition.negated()
589 def _parse_yaml_condition_arch_matches(self, d, key: str, orig_definition_source: str) -> ManifestCondition:
590 arch_matches_raw = self._required_key(d, key, orig_definition_source, expected_type=str)
591 package_state = self.current_binary_package_state
592 arch_matches_as_list = arch_matches_raw.split()
594 if not arch_matches_as_list:
595 _error(f'The condition at {orig_definition_source} must not be empty')
596 if package_state.binary_package.is_arch_all:
597 if arch_matches_as_list[0].startswith('['):
598 arch_matches_as_list[0] = arch_matches_as_list[0][1:]
599 if arch_matches_as_list[-1].endswith(']'):
600 arch_matches_as_list[-1] = arch_matches_as_list[-1][:-1]
601 dpkg_arch_query = self._dpkg_arch_query_table
602 result = dpkg_arch_query.architecture_is_concerned('all', arch_matches_as_list)
603 _error(f'The package architecture restriction at {orig_definition_source} is applied to the'
604 f' "Architecture: all" package {package_state.binary_package.name}, which does not make sense as'
605 f' the condition will always resolves to `{str(result).lower()}`.')
606 if arch_matches_as_list[0].startswith('[') or arch_matches_as_list[-1].endswith(']'):
607 _error(f'The architecture match at {orig_definition_source} must be defined without enclosing it with '
608 '"[" or/and "]" brackets')
609 return ArchMatchManifestCondition(arch_matches_as_list)
611 def _parse_yaml_condition_build_profiles_matches(self,
612 d,
613 key: str,
614 orig_definition_source: str,
615 ) -> ManifestCondition:
616 build_profile_spec = self._required_key(d, key, orig_definition_source, expected_type=str)
617 build_profile_spec = build_profile_spec.strip()
618 if not build_profile_spec:
619 _error(f'The condition at {orig_definition_source} must not be empty')
620 try:
621 active_profiles_match(build_profile_spec, frozenset())
622 except ValueError as e:
623 _error(f'Could not parse the build specification at {orig_definition_source}: {e.args[0]}')
624 return BuildProfileMatch(build_profile_spec)
626 def _parse_yaml_condition(self,
627 d,
628 orig_definition_source: str,
629 ) -> ManifestCondition:
631 supported_mapping_conditions: Dict[str, Callable[[Dict, str, str], ManifestCondition]] = {
632 'not': self._parse_yaml_condition_not,
633 'arch-matches': self._parse_yaml_condition_arch_matches,
634 'build-profiles-matches': self._parse_yaml_condition_build_profiles_matches,
635 }
636 supported_str_conditions: Dict[str, ManifestCondition] = {
637 'cross-compiling': ManifestCondition.is_cross_building(),
638 'can-execute-compiled-binaries': ManifestCondition.can_execute_compiled_binaries(),
639 'run-build-time-tests': ManifestCondition.run_build_time_tests(),
640 }
641 condition = None
642 if isinstance(d, str):
643 condition_key = d
644 condition = supported_str_conditions.get(condition_key)
645 else:
646 condition_key = self._unpack_single_key_dict(d, orig_definition_source)
648 handler = supported_mapping_conditions.get(condition_key)
649 if handler is not None:
650 condition = handler(d, condition_key, orig_definition_source)
652 if condition is None:
653 if condition_key in supported_mapping_conditions:
654 assert not isinstance(d, dict)
655 _error(f'Invalid definition at {orig_definition_source}. The condition {condition_key} must be a '
656 'mapping.')
657 elif condition_key in supported_str_conditions:
658 _error(f'Invalid definition at {orig_definition_source}. The condition {condition_key} must be a '
659 'string.')
660 dict_options = (f'{k} (mapping)' for k in supported_mapping_conditions)
661 string_options = (f'{k} (string)' for k in supported_str_conditions)
662 valid_options = ", ".join(sorted(itertools.chain(dict_options, string_options)))
663 _error(f'Unknown or unsupported condition "{condition_key}" at {orig_definition_source}.'
664 f' Valid conditions are: {valid_options}')
666 return condition
668 def _parse_yaml_when_if_present(self,
669 d,
670 orig_definition_source: str,
671 ) -> Optional[ManifestCondition]:
672 raw_condition = self._optional_key(d,
673 'when',
674 orig_definition_source,
675 expected_type=dict,
676 default_value=None)
677 if raw_condition is None:
678 return None
679 definition_source = f'{orig_definition_source}.when'
680 return self._parse_yaml_condition(raw_condition, definition_source)
682 def _parse_yaml_transformations(self,
683 d,
684 transformations: List[TransformationRule],
685 orig_definition_source: str) -> None:
686 raw_transformations = self._optional_key(d,
687 'transformations',
688 orig_definition_source,
689 expected_type=list,
690 default_value=[])
692 supported_transformations: Dict[str, Callable[[Dict, str, str], TransformationRule]] = {
693 'exclude': self._parse_transformation_exclusion,
694 'move': self._parse_transformation_move,
695 }
697 for no, raw_transformation in enumerate(raw_transformations):
698 definition_source = f'{orig_definition_source}.transformations[{no}]'
699 transformation_key = self._unpack_single_key_dict(raw_transformation, definition_source)
700 handler = supported_transformations.get(transformation_key)
701 if handler is not None:
702 transformation = handler(raw_transformation, transformation_key, definition_source)
703 else:
704 _error(f'Unknown or unsupported transformation "{transformation_key}" at {definition_source}.'
705 ' Transformations must have one of the following keys:'
706 f' {", ".join(sorted(supported_transformations))}')
707 transformations.append(transformation)
709 def _parse_conffile_prior_version_and_owning_package(self,
710 d,
711 definition_source,
712 ) -> Tuple[Optional[str], Optional[str]]:
713 prior_version = self._optional_key(d,
714 MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION,
715 definition_source,
716 expected_type=str,
717 )
718 owning_package = self._optional_key(d,
719 MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE,
720 definition_source,
721 expected_type=str,
722 )
724 if prior_version is not None and not PKGVERSION_REGEX.match(prior_version):
725 _error(f'The {MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION} parameter in {definition_source} must be a valid'
726 f' package version (i.e., match {PKGVERSION_REGEX}).')
727 if owning_package is not None and not PKGNAME_REGEX.match(owning_package):
728 _error(
729 f'The {MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE} parameter in {definition_source} must be a valid'
730 f' package name (i.e., match {PKGNAME_REGEX}).')
732 return prior_version, owning_package
734 def _parse_filename(self, d, key, key_prefix,
735 key_hint=None,
736 required: bool = True,
737 with_prefix: bool = True,
738 ) -> Tuple[str, str]:
739 if required:
740 path = self._required_key(d, key, key_prefix, expected_type=str)
741 else:
742 path = self._optional_key(d, key, key_prefix, expected_type=str)
743 normalized_path = _normalize_path(path, with_prefix=False)
744 if path.endswith('/') or normalized_path == '.':
745 definition_source = f'{key_prefix}.{key}'
746 if key_hint is not None:
747 definition_source += f' <{key_hint}>'
748 if normalized_path == '.':
749 _error(f'The path "{path}" in {definition_source} looks like the root directory,'
750 f' but this feature can only be used for files')
751 _error(f'The path "{path}" in {definition_source} ends with "/" implying it is a directory,'
752 f' but this feature can only be used for files')
753 return path, normalized_path
755 def _parse_conffile_removal(self, d: Dict, definition_source: str) -> DpkgMaintscriptHelperCommand:
756 remove_def = self._required_key(d, MK_CONFFILE_MANAGEMENT_REMOVE, definition_source, expected_type=dict)
757 definition_source += f'.{MK_CONFFILE_MANAGEMENT_REMOVE}'
758 orig_path, normalized_path = self._parse_filename(remove_def,
759 MK_CONFFILE_MANAGEMENT_REMOVE_PATH,
760 definition_source,
761 with_prefix=False
762 )
763 normalized_path = '/' + normalized_path
764 version, owning_package = self._parse_conffile_prior_version_and_owning_package(remove_def, definition_source)
765 definition_source += f' <{orig_path}>'
766 return DpkgMaintscriptHelperCommand.rm_conffile(
767 definition_source,
768 normalized_path,
769 version,
770 owning_package,
771 )
773 def _parse_conffile_rename(self, d: Dict, definition_source: str) -> DpkgMaintscriptHelperCommand:
774 rename_def = self._required_key(d, MK_CONFFILE_MANAGEMENT_RENAME, definition_source, expected_type=dict)
775 definition_source += f'.{MK_CONFFILE_MANAGEMENT_RENAME}'
776 orig_source, normalized_source = self._parse_filename(rename_def,
777 MK_CONFFILE_MANAGEMENT_RENAME_SOURCE,
778 definition_source,
779 with_prefix=False,
780 )
782 orig_target, normalized_target = self._parse_filename(rename_def,
783 MK_CONFFILE_MANAGEMENT_RENAME_TARGET,
784 definition_source,
785 key_hint=orig_source,
786 with_prefix=False,
787 )
788 normalized_source = '/' + normalized_source
789 normalized_target = '/' + normalized_target
791 if normalized_source == normalized_target:
792 _error(f"Invalid rename defined in {definition_source}: The source and target path are the same!")
794 version, owning_package = self._parse_conffile_prior_version_and_owning_package(rename_def, definition_source)
795 definition_source += f' <{orig_source}>'
796 return DpkgMaintscriptHelperCommand.mv_conffile(
797 definition_source,
798 normalized_source,
799 normalized_target,
800 version,
801 owning_package,
802 )
804 def _parse_yaml_conffile_management(self, d, definition_source: str) -> None:
805 entries = self._optional_key(d, MK_CONFFILE_MANAGEMENT, definition_source, expected_type=list, default_value=[])
806 package_state = self.current_binary_package_state
808 supported_conffile_actions: Dict[str, Callable[[Dict, str], DpkgMaintscriptHelperCommand]] = {
809 MK_CONFFILE_MANAGEMENT_REMOVE: self._parse_conffile_removal,
810 MK_CONFFILE_MANAGEMENT_RENAME: self._parse_conffile_rename,
811 }
813 for no, entry in enumerate(entries):
814 path_definition_source = f'{definition_source}.{MK_CONFFILE_MANAGEMENT}[{no}]'
815 key = self._unpack_single_key_dict(entry, path_definition_source)
816 handler = supported_conffile_actions.get(key)
817 if handler is not None:
818 dpkg_script_snippet = handler(entry, path_definition_source)
819 else:
820 _error(f'Unknown or unsupported conffile management "{key}" at {path_definition_source}.'
821 ' Conffile management must have one of the following keys:'
822 f' {", ".join(sorted(entry))}')
823 package_state.dpkg_maintscript_helper_snippets.append(dpkg_script_snippet)
825 def from_yaml_dict(self, yaml_data):
826 manifest_version = self._required_key(yaml_data, MK_MANIFEST_VERSION, '', expected_type=str)
827 if manifest_version not in SUPPORTED_MANIFEST_VERSIONS:
828 _error('Unsupported manifest-version. This implementation supports the following versions:'
829 f' {", ".join(repr(v) for v in SUPPORTED_MANIFEST_VERSIONS)}"')
831 packages_dict = self._optional_key(yaml_data, MK_PACKAGES, '', expected_type=dict)
832 for k, v in packages_dict.items():
833 self._ensure_value_is_type(v, dict, k, MK_PACKAGES)
834 definition_source = f'packages.{k}'
835 with self.binary_package_context(k) as package_state:
836 binary_version = self._optional_key(v,
837 'binary-version',
838 definition_source,
839 expected_type=str,
840 )
841 if binary_version is not None:
842 package_state.binary_version = package_state.substitution.substitute(
843 binary_version,
844 f"{definition_source}.binary-version",
845 )
846 # Parse directories first. It makes it easier for us to get the implicit rules to make sense
847 # for people and saves us from doing it conflict resolution between implicit and explicit
848 # directory rules.
849 self._parse_yaml_directories(v, definition_source)
850 self._parse_yaml_path_metadata(v, definition_source)
851 self._parse_yaml_create_symlinks(v, definition_source)
852 self._parse_yaml_transformations(v, package_state.transformations, definition_source)
853 self._parse_yaml_conffile_management(v, definition_source)
854 return self.build_manifest()
856 def parse_manifest(self) -> HighLevelManifest:
857 with open(self.manifest_path) as fd:
858 data = MANIFEST_YAML.load(fd)
859 self._mutable_yaml_manifest = MutableYAMLManifest(data)
860 return self.from_yaml_dict(data)
863class InstallLikeManifestParser(HighLevelManifestParser):
865 def _parse_owner_keyword(self,
866 value: str,
867 line_no: int):
868 owner = 'root'
869 uid = 0
870 group = 'root'
871 gid = 0
872 if ':' not in value and value != ':':
873 _error('The owner keyword must have [user]:[group] as value and at least one of [user]'
874 f' or [group] must be non-empty (manifest: {self.manifest_path}:{line_no})')
875 if value.count(':') > 1:
876 _error('The owner keyword must have [user]:[group] where none of [user] nor [group] can'
877 f' contain a colon (manifest: {self.manifest_path}:{line_no})')
878 declared_owner, declared_group = value.split(':')
879 if declared_owner:
880 try:
881 owner, uid = self.parse_and_resolve_owner(declared_owner)
882 except ValueError as e:
883 _error('Invalid ownership (user) information in "owner"'
884 f' (manifest: {self.manifest_path}:{line_no}): {e.args[0]}')
885 if declared_group:
886 try:
887 group, gid = self.parse_and_resolve_group(declared_group)
888 except ValueError as e:
889 _error('Invalid ownership (group) information in "owner"'
890 f' (manifest: {self.manifest_path}:{line_no}): {e.args[0]}')
891 return owner, uid, group, gid
893 def _parse_install_line(self,
894 line_no: int,
895 install_line: List[str]
896 ) -> ManifestPathRule:
897 mode = None
898 owner = 'root'
899 uid = 0
900 group = 'root'
901 gid = 0
902 seen_key_words = set()
903 while install_line:
904 if '=' not in install_line[-1]:
905 break
907 latest_entry = install_line.pop()
908 k, v = _parse_keyword(latest_entry)
909 if k in seen_key_words:
910 _error(f'The {k} keyword can only appear once per line'
911 f' (manifest: {self.manifest_path}:{line_no})')
912 seen_key_words.add(k)
913 if k == 'mode':
914 mode = int(v, 8)
915 elif k == 'owner':
916 owner, uid, group, gid = self._parse_owner_keyword(v, line_no)
917 else:
918 _error(f"Unsupported key {k} in {self.manifest_path} line {line_no}")
920 # TODO: Pop the last path and use it as a target dir for true dh_install compat
921 if len(install_line) != 1:
922 # Cannot be bothered to do the "install foo and bar into baz" case right now.
923 _error("There must be exactly one path per line in this prototype (restriction will be loosen later)."
924 f" Bad line is {line_no} in {self.manifest_path}")
926 definition_source = f"line {line_no}"
928 return ManifestPathRule.ensure_metadata(
929 definition_source,
930 self.parse_path_or_glob(install_line[0], definition_source),
931 mode=mode,
932 dirmode=mode,
933 special_mode=mode,
934 uid=uid,
935 uname=owner,
936 gid=gid,
937 gname=group,
938 )
940 def from_install(self, d: List[Tuple[int, List[str]]]):
941 for line_no, install_line in d:
942 path_info = self._parse_install_line(line_no, install_line)
943 self._add_entry(path_info)
945 return self.build_manifest()
947 def parse_manifest(self) -> HighLevelManifest:
948 with open(self.manifest_path) as fd:
949 data = [(no, line.split())
950 for no, line in enumerate(fd, start=1) if line.strip() and not line.strip().startswith("#")
951 ]
952 return self.from_install(data)
955class MTreeManifestParser(HighLevelManifestParser):
957 def _process_mtree_keywords(self,
958 path: str,
959 keywords: Dict[str, str],
960 line_no: int,
961 ) -> ManifestPathRule:
962 baseline = DEFAULT_PATH_INFO
963 keywords.setdefault('mode', oct(baseline.mode)[2:] if baseline.mode is not None else None)
964 if 'uname' in keywords or 'uid' in keywords:
965 keywords.setdefault('uname', '')
966 keywords.setdefault('uid', '')
967 else:
968 keywords.setdefault('uname', baseline.uname)
969 keywords.setdefault('uid', str(baseline.uid))
971 if 'gname' in keywords or 'gid' in keywords:
972 keywords.setdefault('gname', '')
973 keywords.setdefault('gid', '')
974 else:
975 keywords.setdefault('gname', baseline.gname)
976 keywords.setdefault('gid', str(baseline.gid))
978 mtree_mode = keywords['mode']
979 mtree_uname = keywords.get('uname')
980 mtree_uid = keywords['uid']
981 mtree_gname = keywords['gname']
982 mtree_gid = keywords['gid']
984 try:
985 mode = int(mtree_mode, 8) if mtree_mode is not None else None
986 except ValueError as e:
987 _error(f'The tool only supports an octal mode (manifest: {self.manifest_path}:{line_no}): {e.args[0]}')
989 try:
990 owner, uid = self.parse_and_resolve_owner(f'{mtree_uname}:{mtree_uid}')
991 except ValueError as e:
992 _error('Invalid ownership (user) information in "owner"'
993 f' (manifest: {self.manifest_path}:{line_no}): {e.args[0]}')
994 try:
995 group, gid = self.parse_and_resolve_group(f'{mtree_gname}:{mtree_gid}')
996 except ValueError as e:
997 _error('Invalid ownership (group) information in "owner"'
998 f' (manifest: {self.manifest_path}:{line_no}): {e.args[0]}')
999 definition_source = f"line {line_no}"
1000 return ManifestPathRule.ensure_metadata(
1001 definition_source,
1002 self.parse_path_or_glob(path, definition_source),
1003 mode=mode,
1004 dirmode=mode,
1005 special_mode=mode,
1006 uid=uid,
1007 uname=owner,
1008 gid=gid,
1009 gname=group,
1010 )
1012 def _decode_mtree_path(self, mtree_path: str, line_no: int) -> str:
1013 if '\\' in mtree_path:
1014 _error(f"Sorry, mtree path decoding not implemented yet (manifest: {self.manifest_path}:{line_no})")
1015 return mtree_path
1017 def from_mtree(self, d: List[Tuple[int, List[str]]]):
1018 supported_keywords = {
1019 'mode',
1020 'uname',
1021 'uid',
1022 'gname',
1023 'gid',
1024 }
1025 for line_no, install_line in d:
1026 if install_line[0].startswith('/'):
1027 _error("Special commands such as /set and /unset are unsupported. If you wanted a path,"
1028 f" please remove the leading slash. Problematic line is {line_no} in {self.manifest_path}")
1029 if '/' not in install_line[0]:
1030 _error('Only "Full" paths are supported at the moment meaning they have to contain (but must not start'
1031 ' with) a slash. You can prefix the path with ./ or (for directories) end with a slash.'
1032 f' Problematic line is {line_no} in {self.manifest_path}')
1034 path = self._decode_mtree_path(install_line[0], line_no)
1035 keywords = dict(_parse_keyword(kw) for kw in install_line[1:])
1036 unknown_keywords = keywords.keys() - supported_keywords
1037 if unknown_keywords:
1038 as_str = ", ".join(sorted(unknown_keywords))
1039 _error(f'The following keywords in {self.manifest_path} line {line_no} are currently not supported:'
1040 f' {as_str}')
1042 path_info = self._process_mtree_keywords(path, keywords, line_no)
1043 self._add_entry(path_info)
1045 return self.build_manifest()
1047 def parse_manifest(self) -> HighLevelManifest:
1048 with open(self.manifest_path) as fd:
1049 data = [(no, line.split())
1050 for no, line in enumerate(fd, start=1) if line.strip() and not line.strip().startswith("#")
1051 ]
1052 return self.from_mtree(data)
1055def parse_manifest(manifest_path: Optional[str],
1056 binary_packages: Mapping[str, BinaryPackage],
1057 substitution: Substitution,
1058 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
1059 dpkg_arch_query_table: DpkgArchTable,
1060 build_env: DebOptionsAndProfiles,
1061 ) -> HighLevelManifest:
1062 if manifest_path is None:
1063 manifest_path = 'debian/debputy.manifest'
1064 if not os.path.isfile(manifest_path):
1065 return YAMLManifestParser(manifest_path,
1066 binary_packages,
1067 substitution,
1068 dpkg_architecture_variables,
1069 dpkg_arch_query_table,
1070 build_env,
1071 ).build_manifest()
1072 if manifest_path.endswith(('/debputy.manifest', '.yaml')) or manifest_path == 'debputy.manifest':
1073 return YAMLManifestParser(manifest_path,
1074 binary_packages,
1075 substitution,
1076 dpkg_architecture_variables,
1077 dpkg_arch_query_table,
1078 build_env,
1079 ).parse_manifest()
1080 if manifest_path.endswith(('.mtree', '/mtree')) or manifest_path == 'mtree':
1081 return MTreeManifestParser(manifest_path,
1082 binary_packages,
1083 substitution,
1084 dpkg_architecture_variables,
1085 dpkg_arch_query_table,
1086 build_env,
1087 ).parse_manifest()
1088 if manifest_path.endswith(('/install-manifest', '.install-manifest')) or manifest_path == 'install-manifest':
1089 return InstallLikeManifestParser(manifest_path,
1090 binary_packages,
1091 substitution,
1092 dpkg_architecture_variables,
1093 dpkg_arch_query_table,
1094 build_env,
1095 ).parse_manifest()
1096 _error("The manifest path must be either a debputy.manifest, .yaml, a .mtree or a .install-manifest file")