Coverage for debputy/highlevel_manifest.py: 0%
333 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1import dataclasses
2from collections.abc import MutableSequence
3from typing import List, Dict, Iterable, Tuple, Mapping, Set, Any, Union, Optional, Iterator, overload, \
4 TypeVar, Generic
6from debian.debian_support import DpkgArchTable
7from ruamel.yaml import YAML
8from ruamel.yaml.comments import CommentedMap
10from ._deb_options_profiles import DebOptionsAndProfiles
11from ._manifest_constants import *
12from .architecture_support import DpkgArchitectureBuildProcessValuesTable
13from .builtin_manifest_rules import DEFAULT_PATH_INFO
14from .filesystem_scan import FSPath, build_fs_from_root_dir
15from .intermediate_manifest import TarMember, PathType
16from .maintscript_snippet import MaintscriptSnippet, DpkgMaintscriptHelperCommand
17from .manifest_conditions import ConditionContext
18from .manifest_path_rules import ManifestPathRule, ManifestPathInfoRuleType, EnsureDirectoryPathRule
19from .packages import BinaryPackage
20from .path_matcher import MatchRule, PathMatcher
21from .substitution import Substitution
22from .transformation_rules import TransformationRule
23from .util import _error
26MANIFEST_YAML = YAML()
29@dataclasses.dataclass
30class ChangeDescription:
31 match_rule: MatchRule
32 path_info: ManifestPathRule
34 def _pretty_rule_def(self) -> str:
35 if self.path_info.is_builtin_definition:
36 source = f"(builtin rule: {self.path_info.definition_source})"
37 elif self.path_info.is_implicit_definition:
38 source = f"(internal - triggered by {self.path_info.definition_source})"
39 else:
40 source = f"(from {self.path_info.definition_source})"
41 return f'"{self.match_rule.describe_match_short()}" {source}'
43 def _pretty_format_ensure_metadata(self) -> str:
44 attributes = [
45 f' - {self._pretty_rule_def()}',
46 f' path metadata will be set to:',
47 f' owner: {self.path_info.uname} (uid: {self.path_info.uid})',
48 f' group: {self.path_info.gname} (gid: {self.path_info.gid})',
49 ]
50 if self.path_info.mode == self.path_info.dirmode == self.path_info.special_mode:
51 filemode = oct(self.path_info.mode)[2:] if self.path_info.mode is not None else '(unchanged)'
52 attributes.append(f' mode: {filemode}')
53 else:
54 filemode = oct(self.path_info.mode)[2:] if self.path_info.mode is not None else '(unchanged)'
55 dirmode = oct(self.path_info.dirmode)[2:] if self.path_info.dirmode is not None else '(unchanged)'
56 special_mode = oct(self.path_info.special_mode)[
57 2:] if self.path_info.special_mode is not None else '(unchanged)'
58 attributes.append(f' filemode: {filemode} [when matching a file]')
59 attributes.append(f' dirmode: {dirmode} [when matching a file]')
60 attributes.append(f' other mode: {special_mode} [when matching non-file/directories]')
62 return "\n".join(attributes)
64 def _pretty_format_remove_path(self) -> str:
65 return f' - {self._pretty_rule_def()}'
67 def _pretty_format_create_symlink(self) -> str:
68 attributes = [
69 f' - {self._pretty_rule_def()}',
70 f' link target: {self.path_info.link_target}',
71 ]
72 return "\n".join(attributes)
74 def _pretty_format_create_directory(self) -> str:
75 attributes = [
76 f' - {self._pretty_rule_def()}',
77 ]
78 if self.path_info.uid != 0 or self.path_info.gid != 0 or self.path_info.mode != 0o755:
79 attributes.append(f' owner: {self.path_info.uname} (uid: {self.path_info.uid})')
80 attributes.append(f' group: {self.path_info.gname} (gid: {self.path_info.gid})')
81 attributes.append(f' mode: {oct(self.path_info.mode)[2:]}')
82 else:
83 attributes.append(f' default owner and mode (root:root and mode 0755)')
84 return "\n".join(attributes)
86 def pretty_format(self) -> str:
87 rule_type = self.path_info.rule_type
88 if rule_type == ManifestPathInfoRuleType.ENSURE_METADATA:
89 return self._pretty_format_ensure_metadata()
90 elif rule_type == ManifestPathInfoRuleType.EXCLUDE_PATH:
91 return self._pretty_format_remove_path()
92 elif rule_type == ManifestPathInfoRuleType.CREATE_SYMLINK:
93 return self._pretty_format_create_symlink()
94 elif rule_type == ManifestPathInfoRuleType.CREATE_DIRECTORY:
95 return self._pretty_format_create_directory()
96 raise NotImplementedError(f"Missing case for {rule_type}")
99@dataclasses.dataclass(slots=True)
100class PackageTransformationDefinition:
101 binary_package: BinaryPackage
102 binary_version: Optional[str]
103 ensure_directories: Dict[str, EnsureDirectoryPathRule]
104 create_virtual_paths: Dict[str, ManifestPathRule]
105 path_specs: Dict[MatchRule, ManifestPathRule]
106 substitution: Substitution
107 dpkg_maintscript_helper_snippets: List[DpkgMaintscriptHelperCommand]
108 maintscript_snippets: Dict[str, List[MaintscriptSnippet]]
109 transformations: List[TransformationRule]
112def _generate_intermediate_manifest(path_matcher: PathMatcher,
113 fs_root: FSPath,
114 clamp_mtime_to: int,
115 condition_context: ConditionContext,
116 ) -> Iterable[TarMember]:
117 symlinks = []
118 for tar_path_obj, path_info in path_matcher.resolve_all(fs_root):
119 if path_info.condition is not None and not path_info.condition.evaluate(condition_context):
120 path_info.mark_used()
121 continue
122 if path_info.should_include_path():
123 tar_member = path_info.generate_tar_member(tar_path_obj, clamp_mtime_to=clamp_mtime_to)
124 if tar_member.path_type == PathType.SYMLINK:
125 symlinks.append(tar_member)
126 continue
127 yield tar_member
128 else:
129 path_info.mark_used()
130 yield from symlinks
133ST = TypeVar('ST')
134T = TypeVar('T')
137class AbstractYAMLSubStore(Generic[ST]):
139 def __init__(self, parent_store: Any, parent_key: Optional[Union[int, str]], store: Optional[ST] = None) -> None:
140 if parent_store is not None and parent_key is not None:
141 try:
142 from_parent_store = parent_store[parent_key]
143 except (KeyError, IndexError):
144 from_parent_store = None
145 if store is not None and from_parent_store is not None and store is not parent_store:
146 raise ValueError("Store is provided but is not the one already in the parent store")
147 if store is None:
148 store = from_parent_store
149 self._parent_store = parent_store
150 self._parent_key = parent_key
151 self._is_detached = parent_key is None or parent_store is None or parent_key not in parent_store
152 assert self._is_detached or store is not None
153 if store is None:
154 store = self._create_new_instance()
155 self._store: ST = store
157 def _create_new_instance(self) -> ST:
158 raise NotImplementedError
160 def create_definition_if_missing(self) -> None:
161 if self._is_detached:
162 self.create_definition()
164 def create_definition(self) -> None:
165 if not self._is_detached:
166 raise RuntimeError("Definition is already present")
167 parent_store = self._parent_store
168 if parent_store is None:
169 raise RuntimeError(f"Definition is not attached to any parent!? ({self.__class__.__name__})")
170 if isinstance(parent_store, list):
171 assert self._parent_key is None
172 self._parent_key = len(parent_store)
173 self._parent_store.append(self._store)
174 else:
175 parent_store[self._parent_key] = self._store
176 self._is_detached = False
178 def remove_definition(self) -> None:
179 self._ensure_attached()
180 del self._parent_store[self._parent_key]
181 if isinstance(self._parent_store, list):
182 self._parent_key = None
183 self._is_detached = True
185 def _ensure_attached(self) -> None:
186 if self._is_detached:
187 raise RuntimeError("The definition has been removed!")
190class AbstractYAMLListSubStore(Generic[T], AbstractYAMLSubStore[List[T]]):
191 def _create_new_instance(self) -> List[T]:
192 return []
195class AbstractYAMLDictSubStore(Generic[T], AbstractYAMLSubStore[Dict[str, T]]):
197 def _create_new_instance(self) -> Dict[str, T]:
198 return CommentedMap()
201class MutableYAMLSymlink(AbstractYAMLDictSubStore[Any]):
203 @classmethod
204 def new_symlink(cls, link_path: str, link_target: str) -> 'MutableYAMLSymlink':
205 return cls(None, None, store=CommentedMap({
206 MK_CREATE_SYMLINKS_LINK_PATH: link_path,
207 MK_CREATE_SYMLINKS_LINK_TARGET: link_target,
208 }))
210 @property
211 def symlink_path(self) -> str:
212 return self._store[MK_CREATE_SYMLINKS_LINK_PATH]
214 @symlink_path.setter
215 def symlink_path(self, path: str) -> None:
216 self._store[MK_CREATE_SYMLINKS_LINK_PATH] = path
218 @property
219 def symlink_target(self) -> Optional[str]:
220 return self._store[MK_CREATE_SYMLINKS_LINK_TARGET]
222 @symlink_target.setter
223 def symlink_target(self, target: str) -> None:
224 self._store[MK_CREATE_SYMLINKS_LINK_TARGET] = target
227class MutableYAMLConffileManagementItem(AbstractYAMLDictSubStore[Any]):
229 @classmethod
230 def rm_conffile(cls,
231 conffile: str,
232 prior_to_version: Optional[str],
233 owning_package: Optional[str],
234 ) -> 'MutableYAMLConffileManagementItem':
235 r = cls(None,
236 None,
237 store=CommentedMap({
238 MK_CONFFILE_MANAGEMENT_REMOVE: CommentedMap({
239 MK_CONFFILE_MANAGEMENT_REMOVE_PATH: conffile
240 })
241 }))
242 r.prior_to_version = prior_to_version
243 r.owning_package = owning_package
244 return r
246 @classmethod
247 def mv_conffile(cls,
248 old_conffile: str,
249 new_conffile: str,
250 prior_to_version: Optional[str],
251 owning_package: Optional[str],
252 ) -> 'MutableYAMLConffileManagementItem':
253 r = cls(None,
254 None,
255 store=CommentedMap({
256 MK_CONFFILE_MANAGEMENT_RENAME: CommentedMap({
257 MK_CONFFILE_MANAGEMENT_RENAME_SOURCE: old_conffile,
258 MK_CONFFILE_MANAGEMENT_RENAME_TARGET: new_conffile,
259 })
260 }))
261 r.prior_to_version = prior_to_version
262 r.owning_package = owning_package
263 return r
265 @property
266 def _container(self) -> Dict[str, Any]:
267 assert len(self._store) == 1
268 return next(iter(self._store.values()))
270 @property
271 def command(self) -> str:
272 assert len(self._store) == 1
273 return next(iter(self._store))
275 @property
276 def obsolete_conffile(self) -> str:
277 if self.command == MK_CONFFILE_MANAGEMENT_REMOVE:
278 return self._container[MK_CONFFILE_MANAGEMENT_REMOVE_PATH]
279 assert self.command == MK_CONFFILE_MANAGEMENT_RENAME
280 return self._container[MK_CONFFILE_MANAGEMENT_RENAME_SOURCE]
282 @obsolete_conffile.setter
283 def obsolete_conffile(self, value: str) -> None:
284 if self.command == MK_CONFFILE_MANAGEMENT_REMOVE:
285 self._container[MK_CONFFILE_MANAGEMENT_REMOVE_PATH] = value
286 else:
287 assert self.command == MK_CONFFILE_MANAGEMENT_RENAME
288 self._container[MK_CONFFILE_MANAGEMENT_RENAME_SOURCE] = value
290 @property
291 def new_conffile(self) -> str:
292 if self.command != MK_CONFFILE_MANAGEMENT_RENAME:
293 raise TypeError(f"The new_conffile attribute is only applicable to command {MK_CONFFILE_MANAGEMENT_RENAME}."
294 f" This is a {self.command}")
295 return self._container[MK_CONFFILE_MANAGEMENT_RENAME_TARGET]
297 @new_conffile.setter
298 def new_conffile(self, value: str) -> None:
299 if self.command != MK_CONFFILE_MANAGEMENT_RENAME:
300 raise TypeError(f"The new_conffile attribute is only applicable to command {MK_CONFFILE_MANAGEMENT_RENAME}."
301 f" This is a {self.command}")
302 self._container[MK_CONFFILE_MANAGEMENT_RENAME_TARGET] = value
304 @property
305 def prior_to_version(self) -> Optional[str]:
306 return self._container.get(MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION)
308 @prior_to_version.setter
309 def prior_to_version(self, value: Optional[str]) -> None:
310 if value is None:
311 try:
312 del self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION]
313 except KeyError:
314 pass
315 else:
316 self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION] = value
318 @property
319 def owning_package(self) -> Optional[str]:
320 return self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION]
322 @owning_package.setter
323 def owning_package(self, value: Optional[str]) -> None:
324 if value is None:
325 try:
326 del self._container[MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE]
327 except KeyError:
328 pass
329 else:
330 self._container[MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE] = value
333class MutableYAMLPackageDefinition(AbstractYAMLDictSubStore):
335 def _list_store(self, key, *, create_if_absent: bool = False) -> Optional[List[Dict[str, Any]]]:
336 if self._is_detached or key not in self._store:
337 if create_if_absent:
338 return None
339 self.create_definition_if_missing()
340 self._store[key] = []
341 return self._store[key]
343 def _insert_item(self, key: str, item: AbstractYAMLDictSubStore) -> None:
344 parent_store = self._list_store(key, create_if_absent=True)
345 assert parent_store is not None
346 if not item._is_detached or (item._parent_store is not None and item._parent_store is not parent_store):
347 raise RuntimeError("Item is already attached or associated with a different container")
348 item._parent_store = parent_store
349 item.create_definition()
351 def add_symlink(self, symlink: MutableYAMLSymlink) -> None:
352 self._insert_item(MK_CREATE_SYMLINKS, symlink)
354 def symlinks(self) -> Iterable[MutableYAMLSymlink]:
355 store = self._list_store(MK_CREATE_SYMLINKS)
356 if store is None:
357 return
358 yield from (MutableYAMLSymlink(store, i) for i in range(len(store)))
360 def conffile_management_items(self) -> Iterable[MutableYAMLConffileManagementItem]:
361 store = self._list_store(MK_CONFFILE_MANAGEMENT)
362 if store is None:
363 return
364 yield from (MutableYAMLConffileManagementItem(store, i)
365 for i in range(len(store))
366 )
368 def add_conffile_management(self, conffile_management_item: MutableYAMLConffileManagementItem) -> None:
369 self._insert_item(MK_CONFFILE_MANAGEMENT, conffile_management_item)
372class MutableYAMLManifest:
374 def __init__(self, store: Any) -> None:
375 self._store = store
377 @classmethod
378 def empty_manifest(cls) -> 'MutableYAMLManifest':
379 return cls(CommentedMap({
380 MK_MANIFEST_VERSION: DEFAULT_MANIFEST_VERSION
381 }))
383 @property
384 def manifest_version(self) -> str:
385 return self._store[MK_MANIFEST_VERSION]
387 @manifest_version.setter
388 def manifest_version(self, version: str) -> None:
389 if version not in SUPPORTED_MANIFEST_VERSIONS:
390 raise ValueError("Unsupported version")
391 self._store[MK_MANIFEST_VERSION] = version
393 def package(self, name: str, *, create_if_absent=True) -> MutableYAMLPackageDefinition:
394 if 'packages' not in self._store:
395 self._store['packages'] = CommentedMap()
396 packages_store = self._store['packages']
397 package = packages_store.get(name)
398 if package is None:
399 if not create_if_absent:
400 raise KeyError(name)
401 assert packages_store is not None
402 d = MutableYAMLPackageDefinition(packages_store, name)
403 d.create_definition()
404 else:
405 d = MutableYAMLPackageDefinition(packages_store, name)
406 return d
408 def write_to(self, fd) -> None:
409 MANIFEST_YAML.dump(self._store, fd)
412class HighLevelManifest:
414 def __init__(self,
415 manifest_path: str,
416 mutable_manifest: Optional[MutableYAMLManifest],
417 binary_packages: Mapping[str, BinaryPackage],
418 substitution: Substitution,
419 package_transformations: Dict[str, PackageTransformationDefinition],
420 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
421 dpkg_arch_query_table: DpkgArchTable,
422 build_env: DebOptionsAndProfiles,
423 ) -> None:
424 self.manifest_path = manifest_path
425 self.mutable_manifest = mutable_manifest
426 self._binary_packages = binary_packages
427 self.substitution = substitution
428 self.package_transformations = package_transformations
429 self._dpkg_architecture_variables = dpkg_architecture_variables
430 self._dpkg_arch_query_table = dpkg_arch_query_table
431 self._build_env = build_env
432 self._used_for = set()
434 @property
435 def active_packages(self) -> Iterable[BinaryPackage]:
436 yield from (p for p in self._binary_packages.values() if p.should_be_acted_on)
438 @property
439 def all_packages(self) -> Iterable[BinaryPackage]:
440 yield from self._binary_packages.values()
442 def change_descriptions(self, package: str) -> Iterable[ChangeDescription]:
443 seen = set()
444 if package not in self.package_transformations:
445 _error(f"Invalid package: {package}")
446 for k, v in self.package_transformations[package].path_specs.items():
447 if k in seen:
448 continue
449 seen.add(k)
450 yield ChangeDescription(
451 match_rule=k,
452 path_info=v,
453 )
455 def package_state_for(self, package: str) -> PackageTransformationDefinition:
456 return self.package_transformations[package]
458 def apply_to_binary_staging_directory(self,
459 package: str,
460 root_dir: str,
461 clamp_mtime_to: int,
462 ) -> Tuple[FSPath, List[TarMember]]:
463 if package in self._used_for:
464 raise ValueError("HighLevelManifest instances should only be applied to one directory (due to usage checks)")
465 if package not in self.package_transformations:
466 raise ValueError(
467 f'The package "{package}" was not relevant for the manifest!?')
468 self._used_for.add(package)
469 context_for = self.package_transformations[package]
470 condition_context = ConditionContext(
471 binary_package=context_for.binary_package,
472 substitution=context_for.substitution,
473 build_env=self._build_env,
474 dpkg_architecture_variables=self._dpkg_architecture_variables,
475 dpkg_arch_query_table=self._dpkg_arch_query_table,
476 )
478 package_transformation = self.package_transformations[package]
479 fs_root = build_fs_from_root_dir(root_dir)
480 for transformation in package_transformation.transformations:
481 transformation.transform_file_system(fs_root, condition_context)
482 path_matcher = PathMatcher.from_dict(package_transformation.path_specs, DEFAULT_PATH_INFO)
483 intermediate_manifest = list(_generate_intermediate_manifest(
484 path_matcher,
485 fs_root,
486 clamp_mtime_to,
487 condition_context,
488 ))
489 self._validate_usage(package_transformation.path_specs, condition_context)
490 return fs_root, intermediate_manifest
492 def _validate_usage(self,
493 path_specs: Dict[MatchRule, ManifestPathRule],
494 condition_context: ConditionContext,
495 ) -> None:
496 for path, path_info in path_specs.items():
497 if not path_info.has_been_used and (
498 path_info.condition is not None and not path_info.condition.evaluate(condition_context)
499 ):
500 continue
501 if not path_info.has_been_used and not path_info.is_builtin_definition:
502 # Only built-in rules use non-str paths at the moment.
503 _error(f'Declaration "{path.describe_match_exact()}" in manifest {self.manifest_path} was present but'
504 f' not used (nothing matched it). If the declaration is not needed, then please remove it.'
505 ' Otherwise, the path is from the deb is missing. The problematic definition is'
506 f' {path_info.definition_source}'
507 )