Coverage for debputy/dh_migration.py: 0%

191 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-22 14:29 +0100

1import dataclasses 

2import os 

3from typing import Sequence, Iterable, Optional, FrozenSet, List, Tuple 

4 

5from debputy.debhelper_emulation import dhe_pkgfile, dhe_filedoublearray, DHConfigFileLine 

6from debputy.highlevel_manifest import HighLevelManifest, MutableYAMLSymlink, MutableYAMLConffileManagementItem 

7from debputy.installdeb_emulation import _validate_rm_mv_conffile 

8from debputy.packages import BinaryPackage 

9from debputy.substitution import NULL_SUBSTITUTION 

10from debputy.util import _error 

11 

12 

13class AcceptableMigrationIssues: 

14 def __init__(self, values: FrozenSet[str]): 

15 self._values = values 

16 

17 def __contains__(self, item: str) -> bool: 

18 return item in self._values or 'ALL' in self._values 

19 

20 

21class UnsupportedFeature(RuntimeError): 

22 

23 @property 

24 def message(self) -> str: 

25 return self.args[0] 

26 

27 @property 

28 def issue_keys(self) -> Optional[Sequence[str]]: 

29 if len(self.args) < 2: 

30 return None 

31 return self.args[1] 

32 

33 

34class ConflictingChange(RuntimeError): 

35 @property 

36 def message(self) -> str: 

37 return self.args[0] 

38 

39 

40@dataclasses.dataclass 

41class FeatureMigration: 

42 tagline: str 

43 successful_changes: int 

44 already_present: int 

45 warnings: Sequence[str] = tuple() 

46 remove_paths_on_success: Sequence[str] = tuple() 

47 

48 @property 

49 def anything_to_do(self) -> bool: 

50 return bool(self.total_changes_involved) 

51 

52 @property 

53 def total_changes_involved(self) -> int: 

54 return self.successful_changes + len(self.warnings) + len(self.remove_paths_on_success) 

55 

56 

57def _dh_config_file(dctrl_bin: BinaryPackage, 

58 basename, 

59 helper_name, 

60 acceptable_migration_issues: AcceptableMigrationIssues, 

61 warnings: List[str], 

62 paths_to_remove: List[str], 

63 ) -> Tuple[Optional[str], Optional[Iterable[DHConfigFileLine]]]: 

64 dh_config_file = dhe_pkgfile(dctrl_bin, basename) 

65 if dh_config_file is None: 

66 return None, None 

67 if os.access(dh_config_file, os.X_OK): 

68 primary_key = f'executable-{helper_name}-config' 

69 if (primary_key in acceptable_migration_issues 

70 or 'any-executable-dh-configs' in acceptable_migration_issues): 

71 warnings.append(f'TODO: MANUAL MIGRATION of executable dh config "{dh_config_file}" required.') 

72 return None, None 

73 raise UnsupportedFeature(f'Executable configuration files not supported (found: {dh_config_file}).', 

74 [primary_key, 'any-executable-dh-configs'], 

75 ) 

76 

77 paths_to_remove.append(dh_config_file) 

78 content = dhe_filedoublearray(dh_config_file, NULL_SUBSTITUTION) 

79 return dh_config_file, content 

80 

81 

82def migrate_maintscript(manifest: HighLevelManifest, 

83 acceptable_migration_issues: AcceptableMigrationIssues, 

84 ) -> Iterable[FeatureMigration]: 

85 mutable_manifest = manifest.mutable_manifest 

86 paths_to_remove = [] 

87 warnings = [] 

88 manifest_changes = 0 

89 already_present = 0 

90 for dctrl_bin in manifest.all_packages: 

91 mainscript_file, content = _dh_config_file(dctrl_bin, 

92 'maintscript', 

93 'dh_installdeb', 

94 acceptable_migration_issues, 

95 warnings, 

96 paths_to_remove, 

97 ) 

98 

99 if mainscript_file is None: 

100 continue 

101 assert content is not None 

102 

103 package_definition = mutable_manifest.package(dctrl_bin.name) 

104 conffiles = {it.obsolete_conffile: it for it in package_definition.conffile_management_items()} 

105 seen_conffiles = set() 

106 

107 for dhe_line in content: 

108 cmd = dhe_line.tokens[0] 

109 if cmd not in {'rm_conffile', 'mv_conffile'}: 

110 raise UnsupportedFeature(f'The dh_installdeb file {mainscript_file} contains the (currently)' 

111 f' unsupported command "{cmd}" on line {dhe_line.line_no}' 

112 f' (line: "{dhe_line.original_line}"') 

113 

114 try: 

115 _, obsolete_conffile, new_conffile, prior_to_version, owning_package = \ 

116 _validate_rm_mv_conffile(dctrl_bin.name, dhe_line) 

117 except ValueError as e: 

118 _error(f"Validation error in {mainscript_file} on line {dhe_line.line_no}. The error was: {e.args[0]}.") 

119 

120 if obsolete_conffile in seen_conffiles: 

121 raise ConflictingChange(f'The {mainscript_file} file defines actions for "{obsolete_conffile}" twice!' 

122 f' Please ensure that it is defined at most once in that file.') 

123 seen_conffiles.add(obsolete_conffile) 

124 

125 if cmd == 'rm_conffile': 

126 item = MutableYAMLConffileManagementItem.rm_conffile( 

127 obsolete_conffile, 

128 prior_to_version, 

129 owning_package, 

130 ) 

131 else: 

132 assert cmd == 'mv_conffile' 

133 item = MutableYAMLConffileManagementItem.mv_conffile( 

134 obsolete_conffile, 

135 new_conffile, 

136 prior_to_version, 

137 owning_package, 

138 ) 

139 

140 existing_def = conffiles.get(item.obsolete_conffile) 

141 if existing_def is not None: 

142 if not ( 

143 item.command == existing_def.command 

144 and item.new_conffile == existing_def.new_conffile 

145 and item.prior_to_version == existing_def.prior_to_version 

146 and item.owning_package == existing_def.owning_package 

147 ): 

148 raise ConflictingChange(f'The maintscript defines the action {item.command} for' 

149 f' "{obsolete_conffile}" in {mainscript_file}, but there is another' 

150 f' conffile management definition for same path defined already (in the' 

151 f' existing manifest or an migration e.g., inside {mainscript_file})') 

152 already_present += 1 

153 continue 

154 

155 package_definition.add_conffile_management(item) 

156 manifest_changes += 1 

157 

158 yield FeatureMigration('dh_installdeb files', 

159 warnings=warnings, 

160 remove_paths_on_success=paths_to_remove, 

161 successful_changes=manifest_changes, 

162 already_present=already_present, 

163 ) 

164 

165 

166def migrate_links_files(manifest: HighLevelManifest, 

167 acceptable_migration_issues: AcceptableMigrationIssues, 

168 ) -> Iterable[FeatureMigration]: 

169 mutable_manifest = manifest.mutable_manifest 

170 paths_to_remove = [] 

171 warnings = [] 

172 manifest_changes = 0 

173 already_present = 0 

174 for dctrl_bin in manifest.all_packages: 

175 links_file, content = _dh_config_file(dctrl_bin, 

176 'links', 

177 'dh_link', 

178 acceptable_migration_issues, 

179 warnings, 

180 paths_to_remove, 

181 ) 

182 

183 if links_file is None: 

184 continue 

185 assert content is not None 

186 

187 package_definition = mutable_manifest.package(dctrl_bin.name) 

188 defined_symlink = {symlink.symlink_path: symlink.symlink_target for symlink in package_definition.symlinks()} 

189 

190 seen_symlinks = set() 

191 

192 for dhe_line in content: 

193 if len(dhe_line.tokens) != 2: 

194 raise UnsupportedFeature(f'The dh_link file {links_file} did not have exactly two paths on line' 

195 f' {dhe_line.line_no} (line: "{dhe_line.original_line}"') 

196 source, target = dhe_line.tokens 

197 if source in seen_symlinks: 

198 # According to #934499, this has happened in the wild already 

199 raise ConflictingChange(f'The {links_file} file defines the link path {source} twice! Please ensure' 

200 ' that it is defined at most once in that file') 

201 seen_symlinks.add(source) 

202 # Symlinks in .links are always considered absolute, but you were not required to have a leading slash. 

203 # However, in the debputy manifest, you can have relative links, so we should ensure it is explicitly 

204 # absolute. 

205 if not target.startswith('/'): 

206 target = '/' + target 

207 existing_target = defined_symlink.get(source) 

208 if existing_target is not None: 

209 if existing_target != target: 

210 raise ConflictingChange(f'The symlink "{source}" points to "{target}" in {links_file}, but there is' 

211 f' another symlink with same path pointing to "{existing_target}" defined' 

212 ' already (in the existing manifest or an migration e.g., inside' 

213 f' {links_file})') 

214 already_present += 1 

215 continue 

216 package_definition.add_symlink(MutableYAMLSymlink.new_symlink(source, target)) 

217 manifest_changes += 1 

218 yield FeatureMigration('dh_link files', 

219 warnings=warnings, 

220 remove_paths_on_success=paths_to_remove, 

221 successful_changes=manifest_changes, 

222 already_present=already_present, 

223 ) 

224 

225 

226def _print_migration_summary(migrations: List[FeatureMigration], permit_destructive_changes: bool) -> None: 

227 warning_count = 0 

228 print("MIGRATION SUMMERY") 

229 print("=================") 

230 for migration in migrations: 

231 print() 

232 if not migration.anything_to_do: 

233 print(f'Migration: {migration.tagline} - Nothing to do!') 

234 continue 

235 underline = '-' * len(migration.tagline) 

236 print(f'Summary for migration: {migration.tagline}') 

237 print(f'-----------------------{underline}') 

238 print() 

239 if migration.warnings: 

240 print(" /!\\ ATTENTION /!\\") 

241 warning_count += len(migration.warnings) 

242 for warning in migration.warnings: 

243 print(f" * {warning}") 

244 else: 

245 print(f' No warnings detected / manual conversions needed') 

246 print() 

247 if permit_destructive_changes: 

248 print(f' * Manifest changes applied: {migration.successful_changes}') 

249 print(f' * Settings already present: {migration.already_present}') 

250 print(f' * Files removed (merged): {migration.remove_paths_on_success}') 

251 else: 

252 print(f' * Manifest changes to be applied: {migration.successful_changes}') 

253 print(f' * Settings already present: {migration.already_present}') 

254 print(f' * Files to be removed (merged): {migration.remove_paths_on_success}') 

255 

256 if warning_count: 

257 print() 

258 print(f'/!\\ Total number of warnings or manual migrations required: {warning_count}') 

259 

260 

261def migrate_from_dh(manifest: HighLevelManifest, 

262 acceptable_migration_issues: AcceptableMigrationIssues, 

263 permit_destructive_changes: bool) -> None: 

264 migrations = [] 

265 try: 

266 migrations.extend(migrate_links_files(manifest, acceptable_migration_issues)) 

267 migrations.extend(migrate_maintscript(manifest, acceptable_migration_issues)) 

268 except UnsupportedFeature as e: 

269 msg = f'Unable to migrate automatically due to missing features in debputy. The feature is:' \ 

270 f'\n\n * {e.message}' 

271 keys = e.issue_keys 

272 if keys: 

273 primary_key = keys[0] 

274 alt_keys = '' 

275 if len(keys) > 1: 

276 alt_keys = f' Alternatively you can also use one of: {", ".join(keys[1:])}. Please note that some' \ 

277 ' of these may cover more cases.' 

278 msg += (f"\n\nUse --acceptable-migration-issues={primary_key} to convert this into a warning and try again." 

279 " However, you should only do that if you believe you can replace the functionality manually" 

280 f" or the usage is obsolete / can be removed. {alt_keys}" 

281 ) 

282 _error(msg) 

283 except ConflictingChange as e: 

284 _error("The migration tool detected a conflict data being migrated and data already migrated / in the existing" 

285 "manifest." 

286 f"\n\n * {e.message}" 

287 '\n\nPlease review the situation and resolve the conflict manually.') 

288 

289 _print_migration_summary(migrations, permit_destructive_changes) 

290 manifest_change_count = sum((m.successful_changes for m in migrations), 0) 

291 

292 if not manifest_change_count: 

293 print() 

294 print(f"debputy was not able to find any (supported) migrations that it could perform for you.") 

295 return 

296 

297 with open(manifest.manifest_path + ".new", 'w') as fd: 

298 manifest.mutable_manifest.write_to(fd) 

299 

300 if permit_destructive_changes: 

301 if os.path.isfile(manifest.manifest_path): 

302 os.rename(manifest.manifest_path, manifest.manifest_path + '.orig') 

303 os.rename(manifest.manifest_path + '.new', manifest.manifest_path) 

304 print() 

305 print(f"Updated manifest {manifest.manifest_path}") 

306 else: 

307 print() 

308 print(f'Created draft manifest "{manifest.manifest_path}.new" (rename to "{manifest.manifest_path}"' 

309 ' to activate it)') 

310 

311 if permit_destructive_changes: 

312 for path in (p for m in migrations for p in m.remove_paths_on_success): 

313 os.unlink(path) 

314 else: 

315 c = sum((len(m.remove_paths_on_success) for m in migrations), 0) 

316 print(f"(Kept {c} file/files in place as requested)")