Coverage for debputy/dh_migration.py: 0%
191 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1import dataclasses
2import os
3from typing import Sequence, Iterable, Optional, FrozenSet, List, Tuple
5from debputy.debhelper_emulation import dhe_pkgfile, dhe_filedoublearray, DHConfigFileLine
6from debputy.highlevel_manifest import HighLevelManifest, MutableYAMLSymlink, MutableYAMLConffileManagementItem
7from debputy.installdeb_emulation import _validate_rm_mv_conffile
8from debputy.packages import BinaryPackage
9from debputy.substitution import NULL_SUBSTITUTION
10from debputy.util import _error
13class AcceptableMigrationIssues:
14 def __init__(self, values: FrozenSet[str]):
15 self._values = values
17 def __contains__(self, item: str) -> bool:
18 return item in self._values or 'ALL' in self._values
21class UnsupportedFeature(RuntimeError):
23 @property
24 def message(self) -> str:
25 return self.args[0]
27 @property
28 def issue_keys(self) -> Optional[Sequence[str]]:
29 if len(self.args) < 2:
30 return None
31 return self.args[1]
34class ConflictingChange(RuntimeError):
35 @property
36 def message(self) -> str:
37 return self.args[0]
40@dataclasses.dataclass
41class FeatureMigration:
42 tagline: str
43 successful_changes: int
44 already_present: int
45 warnings: Sequence[str] = tuple()
46 remove_paths_on_success: Sequence[str] = tuple()
48 @property
49 def anything_to_do(self) -> bool:
50 return bool(self.total_changes_involved)
52 @property
53 def total_changes_involved(self) -> int:
54 return self.successful_changes + len(self.warnings) + len(self.remove_paths_on_success)
57def _dh_config_file(dctrl_bin: BinaryPackage,
58 basename,
59 helper_name,
60 acceptable_migration_issues: AcceptableMigrationIssues,
61 warnings: List[str],
62 paths_to_remove: List[str],
63 ) -> Tuple[Optional[str], Optional[Iterable[DHConfigFileLine]]]:
64 dh_config_file = dhe_pkgfile(dctrl_bin, basename)
65 if dh_config_file is None:
66 return None, None
67 if os.access(dh_config_file, os.X_OK):
68 primary_key = f'executable-{helper_name}-config'
69 if (primary_key in acceptable_migration_issues
70 or 'any-executable-dh-configs' in acceptable_migration_issues):
71 warnings.append(f'TODO: MANUAL MIGRATION of executable dh config "{dh_config_file}" required.')
72 return None, None
73 raise UnsupportedFeature(f'Executable configuration files not supported (found: {dh_config_file}).',
74 [primary_key, 'any-executable-dh-configs'],
75 )
77 paths_to_remove.append(dh_config_file)
78 content = dhe_filedoublearray(dh_config_file, NULL_SUBSTITUTION)
79 return dh_config_file, content
82def migrate_maintscript(manifest: HighLevelManifest,
83 acceptable_migration_issues: AcceptableMigrationIssues,
84 ) -> Iterable[FeatureMigration]:
85 mutable_manifest = manifest.mutable_manifest
86 paths_to_remove = []
87 warnings = []
88 manifest_changes = 0
89 already_present = 0
90 for dctrl_bin in manifest.all_packages:
91 mainscript_file, content = _dh_config_file(dctrl_bin,
92 'maintscript',
93 'dh_installdeb',
94 acceptable_migration_issues,
95 warnings,
96 paths_to_remove,
97 )
99 if mainscript_file is None:
100 continue
101 assert content is not None
103 package_definition = mutable_manifest.package(dctrl_bin.name)
104 conffiles = {it.obsolete_conffile: it for it in package_definition.conffile_management_items()}
105 seen_conffiles = set()
107 for dhe_line in content:
108 cmd = dhe_line.tokens[0]
109 if cmd not in {'rm_conffile', 'mv_conffile'}:
110 raise UnsupportedFeature(f'The dh_installdeb file {mainscript_file} contains the (currently)'
111 f' unsupported command "{cmd}" on line {dhe_line.line_no}'
112 f' (line: "{dhe_line.original_line}"')
114 try:
115 _, obsolete_conffile, new_conffile, prior_to_version, owning_package = \
116 _validate_rm_mv_conffile(dctrl_bin.name, dhe_line)
117 except ValueError as e:
118 _error(f"Validation error in {mainscript_file} on line {dhe_line.line_no}. The error was: {e.args[0]}.")
120 if obsolete_conffile in seen_conffiles:
121 raise ConflictingChange(f'The {mainscript_file} file defines actions for "{obsolete_conffile}" twice!'
122 f' Please ensure that it is defined at most once in that file.')
123 seen_conffiles.add(obsolete_conffile)
125 if cmd == 'rm_conffile':
126 item = MutableYAMLConffileManagementItem.rm_conffile(
127 obsolete_conffile,
128 prior_to_version,
129 owning_package,
130 )
131 else:
132 assert cmd == 'mv_conffile'
133 item = MutableYAMLConffileManagementItem.mv_conffile(
134 obsolete_conffile,
135 new_conffile,
136 prior_to_version,
137 owning_package,
138 )
140 existing_def = conffiles.get(item.obsolete_conffile)
141 if existing_def is not None:
142 if not (
143 item.command == existing_def.command
144 and item.new_conffile == existing_def.new_conffile
145 and item.prior_to_version == existing_def.prior_to_version
146 and item.owning_package == existing_def.owning_package
147 ):
148 raise ConflictingChange(f'The maintscript defines the action {item.command} for'
149 f' "{obsolete_conffile}" in {mainscript_file}, but there is another'
150 f' conffile management definition for same path defined already (in the'
151 f' existing manifest or an migration e.g., inside {mainscript_file})')
152 already_present += 1
153 continue
155 package_definition.add_conffile_management(item)
156 manifest_changes += 1
158 yield FeatureMigration('dh_installdeb files',
159 warnings=warnings,
160 remove_paths_on_success=paths_to_remove,
161 successful_changes=manifest_changes,
162 already_present=already_present,
163 )
166def migrate_links_files(manifest: HighLevelManifest,
167 acceptable_migration_issues: AcceptableMigrationIssues,
168 ) -> Iterable[FeatureMigration]:
169 mutable_manifest = manifest.mutable_manifest
170 paths_to_remove = []
171 warnings = []
172 manifest_changes = 0
173 already_present = 0
174 for dctrl_bin in manifest.all_packages:
175 links_file, content = _dh_config_file(dctrl_bin,
176 'links',
177 'dh_link',
178 acceptable_migration_issues,
179 warnings,
180 paths_to_remove,
181 )
183 if links_file is None:
184 continue
185 assert content is not None
187 package_definition = mutable_manifest.package(dctrl_bin.name)
188 defined_symlink = {symlink.symlink_path: symlink.symlink_target for symlink in package_definition.symlinks()}
190 seen_symlinks = set()
192 for dhe_line in content:
193 if len(dhe_line.tokens) != 2:
194 raise UnsupportedFeature(f'The dh_link file {links_file} did not have exactly two paths on line'
195 f' {dhe_line.line_no} (line: "{dhe_line.original_line}"')
196 source, target = dhe_line.tokens
197 if source in seen_symlinks:
198 # According to #934499, this has happened in the wild already
199 raise ConflictingChange(f'The {links_file} file defines the link path {source} twice! Please ensure'
200 ' that it is defined at most once in that file')
201 seen_symlinks.add(source)
202 # Symlinks in .links are always considered absolute, but you were not required to have a leading slash.
203 # However, in the debputy manifest, you can have relative links, so we should ensure it is explicitly
204 # absolute.
205 if not target.startswith('/'):
206 target = '/' + target
207 existing_target = defined_symlink.get(source)
208 if existing_target is not None:
209 if existing_target != target:
210 raise ConflictingChange(f'The symlink "{source}" points to "{target}" in {links_file}, but there is'
211 f' another symlink with same path pointing to "{existing_target}" defined'
212 ' already (in the existing manifest or an migration e.g., inside'
213 f' {links_file})')
214 already_present += 1
215 continue
216 package_definition.add_symlink(MutableYAMLSymlink.new_symlink(source, target))
217 manifest_changes += 1
218 yield FeatureMigration('dh_link files',
219 warnings=warnings,
220 remove_paths_on_success=paths_to_remove,
221 successful_changes=manifest_changes,
222 already_present=already_present,
223 )
226def _print_migration_summary(migrations: List[FeatureMigration], permit_destructive_changes: bool) -> None:
227 warning_count = 0
228 print("MIGRATION SUMMERY")
229 print("=================")
230 for migration in migrations:
231 print()
232 if not migration.anything_to_do:
233 print(f'Migration: {migration.tagline} - Nothing to do!')
234 continue
235 underline = '-' * len(migration.tagline)
236 print(f'Summary for migration: {migration.tagline}')
237 print(f'-----------------------{underline}')
238 print()
239 if migration.warnings:
240 print(" /!\\ ATTENTION /!\\")
241 warning_count += len(migration.warnings)
242 for warning in migration.warnings:
243 print(f" * {warning}")
244 else:
245 print(f' No warnings detected / manual conversions needed')
246 print()
247 if permit_destructive_changes:
248 print(f' * Manifest changes applied: {migration.successful_changes}')
249 print(f' * Settings already present: {migration.already_present}')
250 print(f' * Files removed (merged): {migration.remove_paths_on_success}')
251 else:
252 print(f' * Manifest changes to be applied: {migration.successful_changes}')
253 print(f' * Settings already present: {migration.already_present}')
254 print(f' * Files to be removed (merged): {migration.remove_paths_on_success}')
256 if warning_count:
257 print()
258 print(f'/!\\ Total number of warnings or manual migrations required: {warning_count}')
261def migrate_from_dh(manifest: HighLevelManifest,
262 acceptable_migration_issues: AcceptableMigrationIssues,
263 permit_destructive_changes: bool) -> None:
264 migrations = []
265 try:
266 migrations.extend(migrate_links_files(manifest, acceptable_migration_issues))
267 migrations.extend(migrate_maintscript(manifest, acceptable_migration_issues))
268 except UnsupportedFeature as e:
269 msg = f'Unable to migrate automatically due to missing features in debputy. The feature is:' \
270 f'\n\n * {e.message}'
271 keys = e.issue_keys
272 if keys:
273 primary_key = keys[0]
274 alt_keys = ''
275 if len(keys) > 1:
276 alt_keys = f' Alternatively you can also use one of: {", ".join(keys[1:])}. Please note that some' \
277 ' of these may cover more cases.'
278 msg += (f"\n\nUse --acceptable-migration-issues={primary_key} to convert this into a warning and try again."
279 " However, you should only do that if you believe you can replace the functionality manually"
280 f" or the usage is obsolete / can be removed. {alt_keys}"
281 )
282 _error(msg)
283 except ConflictingChange as e:
284 _error("The migration tool detected a conflict data being migrated and data already migrated / in the existing"
285 "manifest."
286 f"\n\n * {e.message}"
287 '\n\nPlease review the situation and resolve the conflict manually.')
289 _print_migration_summary(migrations, permit_destructive_changes)
290 manifest_change_count = sum((m.successful_changes for m in migrations), 0)
292 if not manifest_change_count:
293 print()
294 print(f"debputy was not able to find any (supported) migrations that it could perform for you.")
295 return
297 with open(manifest.manifest_path + ".new", 'w') as fd:
298 manifest.mutable_manifest.write_to(fd)
300 if permit_destructive_changes:
301 if os.path.isfile(manifest.manifest_path):
302 os.rename(manifest.manifest_path, manifest.manifest_path + '.orig')
303 os.rename(manifest.manifest_path + '.new', manifest.manifest_path)
304 print()
305 print(f"Updated manifest {manifest.manifest_path}")
306 else:
307 print()
308 print(f'Created draft manifest "{manifest.manifest_path}.new" (rename to "{manifest.manifest_path}"'
309 ' to activate it)')
311 if permit_destructive_changes:
312 for path in (p for m in migrations for p in m.remove_paths_on_success):
313 os.unlink(path)
314 else:
315 c = sum((len(m.remove_paths_on_success) for m in migrations), 0)
316 print(f"(Kept {c} file/files in place as requested)")