Coverage for debputy/highlevel_manifest.py: 0%

333 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-22 14:29 +0100

1import dataclasses 

2from collections.abc import MutableSequence 

3from typing import List, Dict, Iterable, Tuple, Mapping, Set, Any, Union, Optional, Iterator, overload, \ 

4 TypeVar, Generic 

5 

6from debian.debian_support import DpkgArchTable 

7from ruamel.yaml import YAML 

8from ruamel.yaml.comments import CommentedMap 

9 

10from ._deb_options_profiles import DebOptionsAndProfiles 

11from ._manifest_constants import * 

12from .architecture_support import DpkgArchitectureBuildProcessValuesTable 

13from .builtin_manifest_rules import DEFAULT_PATH_INFO 

14from .filesystem_scan import FSPath, build_fs_from_root_dir 

15from .intermediate_manifest import TarMember, PathType 

16from .maintscript_snippet import MaintscriptSnippet, DpkgMaintscriptHelperCommand 

17from .manifest_conditions import ConditionContext 

18from .manifest_path_rules import ManifestPathRule, ManifestPathInfoRuleType, EnsureDirectoryPathRule 

19from .packages import BinaryPackage 

20from .path_matcher import MatchRule, PathMatcher 

21from .substitution import Substitution 

22from .transformation_rules import TransformationRule 

23from .util import _error 

24 

25 

26MANIFEST_YAML = YAML() 

27 

28 

29@dataclasses.dataclass 

30class ChangeDescription: 

31 match_rule: MatchRule 

32 path_info: ManifestPathRule 

33 

34 def _pretty_rule_def(self) -> str: 

35 if self.path_info.is_builtin_definition: 

36 source = f"(builtin rule: {self.path_info.definition_source})" 

37 elif self.path_info.is_implicit_definition: 

38 source = f"(internal - triggered by {self.path_info.definition_source})" 

39 else: 

40 source = f"(from {self.path_info.definition_source})" 

41 return f'"{self.match_rule.describe_match_short()}" {source}' 

42 

43 def _pretty_format_ensure_metadata(self) -> str: 

44 attributes = [ 

45 f' - {self._pretty_rule_def()}', 

46 f' path metadata will be set to:', 

47 f' owner: {self.path_info.uname} (uid: {self.path_info.uid})', 

48 f' group: {self.path_info.gname} (gid: {self.path_info.gid})', 

49 ] 

50 if self.path_info.mode == self.path_info.dirmode == self.path_info.special_mode: 

51 filemode = oct(self.path_info.mode)[2:] if self.path_info.mode is not None else '(unchanged)' 

52 attributes.append(f' mode: {filemode}') 

53 else: 

54 filemode = oct(self.path_info.mode)[2:] if self.path_info.mode is not None else '(unchanged)' 

55 dirmode = oct(self.path_info.dirmode)[2:] if self.path_info.dirmode is not None else '(unchanged)' 

56 special_mode = oct(self.path_info.special_mode)[ 

57 2:] if self.path_info.special_mode is not None else '(unchanged)' 

58 attributes.append(f' filemode: {filemode} [when matching a file]') 

59 attributes.append(f' dirmode: {dirmode} [when matching a file]') 

60 attributes.append(f' other mode: {special_mode} [when matching non-file/directories]') 

61 

62 return "\n".join(attributes) 

63 

64 def _pretty_format_remove_path(self) -> str: 

65 return f' - {self._pretty_rule_def()}' 

66 

67 def _pretty_format_create_symlink(self) -> str: 

68 attributes = [ 

69 f' - {self._pretty_rule_def()}', 

70 f' link target: {self.path_info.link_target}', 

71 ] 

72 return "\n".join(attributes) 

73 

74 def _pretty_format_create_directory(self) -> str: 

75 attributes = [ 

76 f' - {self._pretty_rule_def()}', 

77 ] 

78 if self.path_info.uid != 0 or self.path_info.gid != 0 or self.path_info.mode != 0o755: 

79 attributes.append(f' owner: {self.path_info.uname} (uid: {self.path_info.uid})') 

80 attributes.append(f' group: {self.path_info.gname} (gid: {self.path_info.gid})') 

81 attributes.append(f' mode: {oct(self.path_info.mode)[2:]}') 

82 else: 

83 attributes.append(f' default owner and mode (root:root and mode 0755)') 

84 return "\n".join(attributes) 

85 

86 def pretty_format(self) -> str: 

87 rule_type = self.path_info.rule_type 

88 if rule_type == ManifestPathInfoRuleType.ENSURE_METADATA: 

89 return self._pretty_format_ensure_metadata() 

90 elif rule_type == ManifestPathInfoRuleType.EXCLUDE_PATH: 

91 return self._pretty_format_remove_path() 

92 elif rule_type == ManifestPathInfoRuleType.CREATE_SYMLINK: 

93 return self._pretty_format_create_symlink() 

94 elif rule_type == ManifestPathInfoRuleType.CREATE_DIRECTORY: 

95 return self._pretty_format_create_directory() 

96 raise NotImplementedError(f"Missing case for {rule_type}") 

97 

98 

99@dataclasses.dataclass(slots=True) 

100class PackageTransformationDefinition: 

101 binary_package: BinaryPackage 

102 binary_version: Optional[str] 

103 ensure_directories: Dict[str, EnsureDirectoryPathRule] 

104 create_virtual_paths: Dict[str, ManifestPathRule] 

105 path_specs: Dict[MatchRule, ManifestPathRule] 

106 substitution: Substitution 

107 dpkg_maintscript_helper_snippets: List[DpkgMaintscriptHelperCommand] 

108 maintscript_snippets: Dict[str, List[MaintscriptSnippet]] 

109 transformations: List[TransformationRule] 

110 

111 

112def _generate_intermediate_manifest(path_matcher: PathMatcher, 

113 fs_root: FSPath, 

114 clamp_mtime_to: int, 

115 condition_context: ConditionContext, 

116 ) -> Iterable[TarMember]: 

117 symlinks = [] 

118 for tar_path_obj, path_info in path_matcher.resolve_all(fs_root): 

119 if path_info.condition is not None and not path_info.condition.evaluate(condition_context): 

120 path_info.mark_used() 

121 continue 

122 if path_info.should_include_path(): 

123 tar_member = path_info.generate_tar_member(tar_path_obj, clamp_mtime_to=clamp_mtime_to) 

124 if tar_member.path_type == PathType.SYMLINK: 

125 symlinks.append(tar_member) 

126 continue 

127 yield tar_member 

128 else: 

129 path_info.mark_used() 

130 yield from symlinks 

131 

132 

133ST = TypeVar('ST') 

134T = TypeVar('T') 

135 

136 

137class AbstractYAMLSubStore(Generic[ST]): 

138 

139 def __init__(self, parent_store: Any, parent_key: Optional[Union[int, str]], store: Optional[ST] = None) -> None: 

140 if parent_store is not None and parent_key is not None: 

141 try: 

142 from_parent_store = parent_store[parent_key] 

143 except (KeyError, IndexError): 

144 from_parent_store = None 

145 if store is not None and from_parent_store is not None and store is not parent_store: 

146 raise ValueError("Store is provided but is not the one already in the parent store") 

147 if store is None: 

148 store = from_parent_store 

149 self._parent_store = parent_store 

150 self._parent_key = parent_key 

151 self._is_detached = parent_key is None or parent_store is None or parent_key not in parent_store 

152 assert self._is_detached or store is not None 

153 if store is None: 

154 store = self._create_new_instance() 

155 self._store: ST = store 

156 

157 def _create_new_instance(self) -> ST: 

158 raise NotImplementedError 

159 

160 def create_definition_if_missing(self) -> None: 

161 if self._is_detached: 

162 self.create_definition() 

163 

164 def create_definition(self) -> None: 

165 if not self._is_detached: 

166 raise RuntimeError("Definition is already present") 

167 parent_store = self._parent_store 

168 if parent_store is None: 

169 raise RuntimeError(f"Definition is not attached to any parent!? ({self.__class__.__name__})") 

170 if isinstance(parent_store, list): 

171 assert self._parent_key is None 

172 self._parent_key = len(parent_store) 

173 self._parent_store.append(self._store) 

174 else: 

175 parent_store[self._parent_key] = self._store 

176 self._is_detached = False 

177 

178 def remove_definition(self) -> None: 

179 self._ensure_attached() 

180 del self._parent_store[self._parent_key] 

181 if isinstance(self._parent_store, list): 

182 self._parent_key = None 

183 self._is_detached = True 

184 

185 def _ensure_attached(self) -> None: 

186 if self._is_detached: 

187 raise RuntimeError("The definition has been removed!") 

188 

189 

190class AbstractYAMLListSubStore(Generic[T], AbstractYAMLSubStore[List[T]]): 

191 def _create_new_instance(self) -> List[T]: 

192 return [] 

193 

194 

195class AbstractYAMLDictSubStore(Generic[T], AbstractYAMLSubStore[Dict[str, T]]): 

196 

197 def _create_new_instance(self) -> Dict[str, T]: 

198 return CommentedMap() 

199 

200 

201class MutableYAMLSymlink(AbstractYAMLDictSubStore[Any]): 

202 

203 @classmethod 

204 def new_symlink(cls, link_path: str, link_target: str) -> 'MutableYAMLSymlink': 

205 return cls(None, None, store=CommentedMap({ 

206 MK_CREATE_SYMLINKS_LINK_PATH: link_path, 

207 MK_CREATE_SYMLINKS_LINK_TARGET: link_target, 

208 })) 

209 

210 @property 

211 def symlink_path(self) -> str: 

212 return self._store[MK_CREATE_SYMLINKS_LINK_PATH] 

213 

214 @symlink_path.setter 

215 def symlink_path(self, path: str) -> None: 

216 self._store[MK_CREATE_SYMLINKS_LINK_PATH] = path 

217 

218 @property 

219 def symlink_target(self) -> Optional[str]: 

220 return self._store[MK_CREATE_SYMLINKS_LINK_TARGET] 

221 

222 @symlink_target.setter 

223 def symlink_target(self, target: str) -> None: 

224 self._store[MK_CREATE_SYMLINKS_LINK_TARGET] = target 

225 

226 

227class MutableYAMLConffileManagementItem(AbstractYAMLDictSubStore[Any]): 

228 

229 @classmethod 

230 def rm_conffile(cls, 

231 conffile: str, 

232 prior_to_version: Optional[str], 

233 owning_package: Optional[str], 

234 ) -> 'MutableYAMLConffileManagementItem': 

235 r = cls(None, 

236 None, 

237 store=CommentedMap({ 

238 MK_CONFFILE_MANAGEMENT_REMOVE: CommentedMap({ 

239 MK_CONFFILE_MANAGEMENT_REMOVE_PATH: conffile 

240 }) 

241 })) 

242 r.prior_to_version = prior_to_version 

243 r.owning_package = owning_package 

244 return r 

245 

246 @classmethod 

247 def mv_conffile(cls, 

248 old_conffile: str, 

249 new_conffile: str, 

250 prior_to_version: Optional[str], 

251 owning_package: Optional[str], 

252 ) -> 'MutableYAMLConffileManagementItem': 

253 r = cls(None, 

254 None, 

255 store=CommentedMap({ 

256 MK_CONFFILE_MANAGEMENT_RENAME: CommentedMap({ 

257 MK_CONFFILE_MANAGEMENT_RENAME_SOURCE: old_conffile, 

258 MK_CONFFILE_MANAGEMENT_RENAME_TARGET: new_conffile, 

259 }) 

260 })) 

261 r.prior_to_version = prior_to_version 

262 r.owning_package = owning_package 

263 return r 

264 

265 @property 

266 def _container(self) -> Dict[str, Any]: 

267 assert len(self._store) == 1 

268 return next(iter(self._store.values())) 

269 

270 @property 

271 def command(self) -> str: 

272 assert len(self._store) == 1 

273 return next(iter(self._store)) 

274 

275 @property 

276 def obsolete_conffile(self) -> str: 

277 if self.command == MK_CONFFILE_MANAGEMENT_REMOVE: 

278 return self._container[MK_CONFFILE_MANAGEMENT_REMOVE_PATH] 

279 assert self.command == MK_CONFFILE_MANAGEMENT_RENAME 

280 return self._container[MK_CONFFILE_MANAGEMENT_RENAME_SOURCE] 

281 

282 @obsolete_conffile.setter 

283 def obsolete_conffile(self, value: str) -> None: 

284 if self.command == MK_CONFFILE_MANAGEMENT_REMOVE: 

285 self._container[MK_CONFFILE_MANAGEMENT_REMOVE_PATH] = value 

286 else: 

287 assert self.command == MK_CONFFILE_MANAGEMENT_RENAME 

288 self._container[MK_CONFFILE_MANAGEMENT_RENAME_SOURCE] = value 

289 

290 @property 

291 def new_conffile(self) -> str: 

292 if self.command != MK_CONFFILE_MANAGEMENT_RENAME: 

293 raise TypeError(f"The new_conffile attribute is only applicable to command {MK_CONFFILE_MANAGEMENT_RENAME}." 

294 f" This is a {self.command}") 

295 return self._container[MK_CONFFILE_MANAGEMENT_RENAME_TARGET] 

296 

297 @new_conffile.setter 

298 def new_conffile(self, value: str) -> None: 

299 if self.command != MK_CONFFILE_MANAGEMENT_RENAME: 

300 raise TypeError(f"The new_conffile attribute is only applicable to command {MK_CONFFILE_MANAGEMENT_RENAME}." 

301 f" This is a {self.command}") 

302 self._container[MK_CONFFILE_MANAGEMENT_RENAME_TARGET] = value 

303 

304 @property 

305 def prior_to_version(self) -> Optional[str]: 

306 return self._container.get(MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION) 

307 

308 @prior_to_version.setter 

309 def prior_to_version(self, value: Optional[str]) -> None: 

310 if value is None: 

311 try: 

312 del self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION] 

313 except KeyError: 

314 pass 

315 else: 

316 self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION] = value 

317 

318 @property 

319 def owning_package(self) -> Optional[str]: 

320 return self._container[MK_CONFFILE_MANAGEMENT_X_PRIOR_TO_VERSION] 

321 

322 @owning_package.setter 

323 def owning_package(self, value: Optional[str]) -> None: 

324 if value is None: 

325 try: 

326 del self._container[MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE] 

327 except KeyError: 

328 pass 

329 else: 

330 self._container[MK_CONFFILE_MANAGEMENT_X_OWNING_PACKAGE] = value 

331 

332 

333class MutableYAMLPackageDefinition(AbstractYAMLDictSubStore): 

334 

335 def _list_store(self, key, *, create_if_absent: bool = False) -> Optional[List[Dict[str, Any]]]: 

336 if self._is_detached or key not in self._store: 

337 if create_if_absent: 

338 return None 

339 self.create_definition_if_missing() 

340 self._store[key] = [] 

341 return self._store[key] 

342 

343 def _insert_item(self, key: str, item: AbstractYAMLDictSubStore) -> None: 

344 parent_store = self._list_store(key, create_if_absent=True) 

345 assert parent_store is not None 

346 if not item._is_detached or (item._parent_store is not None and item._parent_store is not parent_store): 

347 raise RuntimeError("Item is already attached or associated with a different container") 

348 item._parent_store = parent_store 

349 item.create_definition() 

350 

351 def add_symlink(self, symlink: MutableYAMLSymlink) -> None: 

352 self._insert_item(MK_CREATE_SYMLINKS, symlink) 

353 

354 def symlinks(self) -> Iterable[MutableYAMLSymlink]: 

355 store = self._list_store(MK_CREATE_SYMLINKS) 

356 if store is None: 

357 return 

358 yield from (MutableYAMLSymlink(store, i) for i in range(len(store))) 

359 

360 def conffile_management_items(self) -> Iterable[MutableYAMLConffileManagementItem]: 

361 store = self._list_store(MK_CONFFILE_MANAGEMENT) 

362 if store is None: 

363 return 

364 yield from (MutableYAMLConffileManagementItem(store, i) 

365 for i in range(len(store)) 

366 ) 

367 

368 def add_conffile_management(self, conffile_management_item: MutableYAMLConffileManagementItem) -> None: 

369 self._insert_item(MK_CONFFILE_MANAGEMENT, conffile_management_item) 

370 

371 

372class MutableYAMLManifest: 

373 

374 def __init__(self, store: Any) -> None: 

375 self._store = store 

376 

377 @classmethod 

378 def empty_manifest(cls) -> 'MutableYAMLManifest': 

379 return cls(CommentedMap({ 

380 MK_MANIFEST_VERSION: DEFAULT_MANIFEST_VERSION 

381 })) 

382 

383 @property 

384 def manifest_version(self) -> str: 

385 return self._store[MK_MANIFEST_VERSION] 

386 

387 @manifest_version.setter 

388 def manifest_version(self, version: str) -> None: 

389 if version not in SUPPORTED_MANIFEST_VERSIONS: 

390 raise ValueError("Unsupported version") 

391 self._store[MK_MANIFEST_VERSION] = version 

392 

393 def package(self, name: str, *, create_if_absent=True) -> MutableYAMLPackageDefinition: 

394 if 'packages' not in self._store: 

395 self._store['packages'] = CommentedMap() 

396 packages_store = self._store['packages'] 

397 package = packages_store.get(name) 

398 if package is None: 

399 if not create_if_absent: 

400 raise KeyError(name) 

401 assert packages_store is not None 

402 d = MutableYAMLPackageDefinition(packages_store, name) 

403 d.create_definition() 

404 else: 

405 d = MutableYAMLPackageDefinition(packages_store, name) 

406 return d 

407 

408 def write_to(self, fd) -> None: 

409 MANIFEST_YAML.dump(self._store, fd) 

410 

411 

412class HighLevelManifest: 

413 

414 def __init__(self, 

415 manifest_path: str, 

416 mutable_manifest: Optional[MutableYAMLManifest], 

417 binary_packages: Mapping[str, BinaryPackage], 

418 substitution: Substitution, 

419 package_transformations: Dict[str, PackageTransformationDefinition], 

420 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, 

421 dpkg_arch_query_table: DpkgArchTable, 

422 build_env: DebOptionsAndProfiles, 

423 ) -> None: 

424 self.manifest_path = manifest_path 

425 self.mutable_manifest = mutable_manifest 

426 self._binary_packages = binary_packages 

427 self.substitution = substitution 

428 self.package_transformations = package_transformations 

429 self._dpkg_architecture_variables = dpkg_architecture_variables 

430 self._dpkg_arch_query_table = dpkg_arch_query_table 

431 self._build_env = build_env 

432 self._used_for = set() 

433 

434 @property 

435 def active_packages(self) -> Iterable[BinaryPackage]: 

436 yield from (p for p in self._binary_packages.values() if p.should_be_acted_on) 

437 

438 @property 

439 def all_packages(self) -> Iterable[BinaryPackage]: 

440 yield from self._binary_packages.values() 

441 

442 def change_descriptions(self, package: str) -> Iterable[ChangeDescription]: 

443 seen = set() 

444 if package not in self.package_transformations: 

445 _error(f"Invalid package: {package}") 

446 for k, v in self.package_transformations[package].path_specs.items(): 

447 if k in seen: 

448 continue 

449 seen.add(k) 

450 yield ChangeDescription( 

451 match_rule=k, 

452 path_info=v, 

453 ) 

454 

455 def package_state_for(self, package: str) -> PackageTransformationDefinition: 

456 return self.package_transformations[package] 

457 

458 def apply_to_binary_staging_directory(self, 

459 package: str, 

460 root_dir: str, 

461 clamp_mtime_to: int, 

462 ) -> Tuple[FSPath, List[TarMember]]: 

463 if package in self._used_for: 

464 raise ValueError("HighLevelManifest instances should only be applied to one directory (due to usage checks)") 

465 if package not in self.package_transformations: 

466 raise ValueError( 

467 f'The package "{package}" was not relevant for the manifest!?') 

468 self._used_for.add(package) 

469 context_for = self.package_transformations[package] 

470 condition_context = ConditionContext( 

471 binary_package=context_for.binary_package, 

472 substitution=context_for.substitution, 

473 build_env=self._build_env, 

474 dpkg_architecture_variables=self._dpkg_architecture_variables, 

475 dpkg_arch_query_table=self._dpkg_arch_query_table, 

476 ) 

477 

478 package_transformation = self.package_transformations[package] 

479 fs_root = build_fs_from_root_dir(root_dir) 

480 for transformation in package_transformation.transformations: 

481 transformation.transform_file_system(fs_root, condition_context) 

482 path_matcher = PathMatcher.from_dict(package_transformation.path_specs, DEFAULT_PATH_INFO) 

483 intermediate_manifest = list(_generate_intermediate_manifest( 

484 path_matcher, 

485 fs_root, 

486 clamp_mtime_to, 

487 condition_context, 

488 )) 

489 self._validate_usage(package_transformation.path_specs, condition_context) 

490 return fs_root, intermediate_manifest 

491 

492 def _validate_usage(self, 

493 path_specs: Dict[MatchRule, ManifestPathRule], 

494 condition_context: ConditionContext, 

495 ) -> None: 

496 for path, path_info in path_specs.items(): 

497 if not path_info.has_been_used and ( 

498 path_info.condition is not None and not path_info.condition.evaluate(condition_context) 

499 ): 

500 continue 

501 if not path_info.has_been_used and not path_info.is_builtin_definition: 

502 # Only built-in rules use non-str paths at the moment. 

503 _error(f'Declaration "{path.describe_match_exact()}" in manifest {self.manifest_path} was present but' 

504 f' not used (nothing matched it). If the declaration is not needed, then please remove it.' 

505 ' Otherwise, the path is from the deb is missing. The problematic definition is' 

506 f' {path_info.definition_source}' 

507 )