Coverage for debputy/deb_packaging_support.py: 0%

216 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-22 14:29 +0100

1import contextlib 

2import hashlib 

3import itertools 

4import os 

5import shutil 

6import subprocess 

7from typing import Iterable, List, TextIO, Optional, Literal, Set, Any, Generator 

8 

9from debian.substvars import Substvars 

10 

11from debputy.architecture_support import dpkg_architecture_table 

12from debputy.debhelper_emulation import dhe_install_pkg_file_as_ctrl_file_if_present, dhe_generated_file, \ 

13 dhe_dbgsym_root_dir, dhe_read_dbgsym_build_ids, dhe_read_dbgsym_migration 

14from debputy.filesystem_scan import FSPath 

15from debputy.highlevel_manifest import HighLevelManifest, PackageTransformationDefinition 

16from debputy.installdeb_emulation import dh_installdeb_emulation 

17from debputy.maintscript_snippet import UDEB_CONTROL_SCRIPTS, MaintscriptSnippet, STD_CONTROL_SCRIPTS, \ 

18 ALL_CONTROL_SCRIPTS 

19from debputy.packages import BinaryPackage 

20from debputy.util import debputy_autoscript_tagline, _error, ensure_dir 

21 

22 

23def generate_md5sums_file(root_dir: str, fs_root: FSPath) -> None: 

24 conffiles = os.path.join(root_dir, 'DEBIAN', 'conffiles') 

25 md5sums = os.path.join(root_dir, 'DEBIAN', 'md5sums') 

26 exclude = set() 

27 if os.path.isfile(conffiles): 

28 with open(conffiles, 'rt') as fd: 

29 for line in fd: 

30 if not line.startswith("/"): 

31 continue 

32 exclude.add('.' + line.rstrip("\n")) 

33 had_content = False 

34 files = (path for path in fs_root.all_paths() if path.is_file and path.path not in exclude) 

35 with open(md5sums, 'wt') as md5fd: 

36 for member in files: 

37 path = member.path 

38 assert path.startswith('./') 

39 path = path[2:] 

40 with open(member.fs_path, 'rb') as f: 

41 file_hash = hashlib.md5() 

42 while chunk := f.read(8192): 

43 file_hash.update(chunk) 

44 had_content = True 

45 md5fd.write(f"{file_hash.hexdigest()} {path}\n") 

46 if not had_content: 

47 os.unlink(md5sums) 

48 

49 

50def install_or_generate_conffiles(binary_package: BinaryPackage, root_dir: str, fs_root: FSPath) -> None: 

51 conffiles_dest = os.path.join(root_dir, 'DEBIAN', 'conffiles') 

52 dhe_install_pkg_file_as_ctrl_file_if_present(binary_package, 'conffiles', root_dir, 0o0644) 

53 etc_dir = fs_root.lookup('etc') 

54 if etc_dir: 

55 _add_conffiles(conffiles_dest, (p for p in etc_dir.all_paths() if p.is_file)) 

56 if os.path.isfile(conffiles_dest): 

57 os.chmod(conffiles_dest, 0o0644) 

58 

59 

60def build_dbgsym_package_if_relevant(binary_package: BinaryPackage, 

61 dpkg_deb_args: List[str], 

62 output_dir: str, 

63 ) -> None: 

64 if binary_package.is_udeb: 

65 # We never built udebs due to #797391. If it is easy to do, we can fix it when we do the control file 

66 return 

67 if not os.path.isdir(output_dir): 

68 _error(f"Cannot produce a dbgsym package when output path is not a directory.") 

69 return 

70 dbgsym_root = dhe_dbgsym_root_dir(binary_package) 

71 if not os.path.isdir(dbgsym_root): 

72 return 

73 package = binary_package.name 

74 if not os.path.isfile(os.path.join(dbgsym_root, 'DEBIAN', 'control')): 

75 # Will not be a problem when we integrate generation of DEBIAN/control (#18) 

76 _error(f"There is a dbgsym directory from debhelper for {package}, but not a DEBIAN/control") 

77 

78 dpkg_cmd = [ 

79 'dpkg-deb', 

80 '--root-owner-group', 

81 ] 

82 dpkg_cmd.extend(dpkg_deb_args) 

83 dpkg_cmd.extend([ 

84 '--build', 

85 dbgsym_root, 

86 output_dir, 

87 ]) 

88 

89 print(f"Executing for {package}-dbgsym: {dpkg_cmd}") 

90 subprocess.check_call(dpkg_cmd) 

91 

92 

93def setup_control_files(binary_package: BinaryPackage, 

94 root_dir: str, 

95 fs_root: FSPath, 

96 manifest: HighLevelManifest, 

97 ) -> None: 

98 ensure_dir(os.path.join(root_dir, 'DEBIAN')) 

99 package_state = manifest.package_state_for(binary_package.name) 

100 if binary_package.is_udeb: 

101 _generate_control_files(binary_package, package_state, root_dir, fs_root) 

102 for script in UDEB_CONTROL_SCRIPTS: 

103 dhe_install_pkg_file_as_ctrl_file_if_present(binary_package, script, root_dir, 0o0755) 

104 return 

105 

106 state = manifest.package_state_for(binary_package.name) 

107 if state.maintscript_snippets: 

108 for script, snippets in state.maintscript_snippets.items(): 

109 _insert_autoscript_snippets( 

110 binary_package, 

111 script, 

112 snippets, 

113 debputy_autoscript_tagline(), 

114 ) 

115 

116 dh_installdeb_emulation(binary_package, root_dir) 

117 install_or_generate_conffiles(binary_package, root_dir, fs_root) 

118 generate_md5sums_file(root_dir, fs_root) 

119 _generate_control_files(binary_package, package_state, root_dir, fs_root) 

120 

121 

122def _add_conffiles(conffiles_dest: str, conffile_matches: Iterable[FSPath]) -> None: 

123 with open(conffiles_dest, 'at') as fd: 

124 for conffile_match in conffile_matches: 

125 conffile = conffile_match.path 

126 assert conffile_match.is_file 

127 assert conffile.startswith('./') 

128 fd.write(f"{conffile[1:]}\n") 

129 if os.stat(conffiles_dest).st_size == 0: 

130 os.utime(conffiles_dest) 

131 

132 

133def _add_snippets_to_script(fd: TextIO, 

134 snippets: Iterable[MaintscriptSnippet], 

135 tool_with_version: str, 

136 ) -> None: 

137 

138 fd.write(f"# Automatically added by {tool_with_version}\n") 

139 for snippet in snippets: 

140 fd.write(f'# - source: {snippet.definition_source}\n') 

141 fd.write(snippet.snippet) 

142 if not snippet.snippet.endswith('\n'): 

143 fd.write('\n') 

144 fd.write("# End automatically added section\n") 

145 

146 

147def _insert_by_snippet_type(binary_package: BinaryPackage, 

148 script: str, 

149 snippet_order: Optional[Literal['service']], 

150 snippets: List[MaintscriptSnippet], 

151 reverse_order: bool, 

152 tool_with_version: str, 

153 ) -> None: 

154 if snippet_order is None: 

155 outfile = f'debian/{binary_package.name}.{script}.debhelper' 

156 else: 

157 outfile = dhe_generated_file(binary_package, f"{script}.{snippet_order}") 

158 

159 if reverse_order: 

160 with open(outfile, 'wt') as wfd, open(outfile, 'rt') as rfd: 

161 _add_snippets_to_script(wfd, reversed(snippets), tool_with_version) 

162 shutil.copyfileobj(rfd, wfd) 

163 else: 

164 with open(outfile, 'at') as fd: 

165 _add_snippets_to_script(fd, snippets, tool_with_version) 

166 

167 

168def _insert_autoscript_snippets(binary_package: BinaryPackage, 

169 script: str, 

170 snippets: List[MaintscriptSnippet], 

171 tool_with_version: str, 

172 ) -> None: 

173 

174 assert script in STD_CONTROL_SCRIPTS 

175 reverse_order = script in ('postrm', 'prerm') 

176 general_snippets = [s for s in snippets if s.snippet_order is None] 

177 service_snippets = [s for s in snippets if s.snippet_order == 'service'] 

178 if general_snippets: 

179 _insert_by_snippet_type( 

180 binary_package, 

181 script, 

182 None, 

183 snippets, 

184 reverse_order, 

185 tool_with_version 

186 ) 

187 if service_snippets: 

188 _insert_by_snippet_type( 

189 binary_package, 

190 script, 

191 'service', 

192 snippets, 

193 reverse_order, 

194 tool_with_version 

195 ) 

196 

197 

198@contextlib.contextmanager 

199def _ensure_base_substvars_defined(substvars_file: str) -> Generator[Substvars, Any, Any]: 

200 with Substvars.load_from_path(substvars_file, missing_ok=True) as substvars: 

201 for substvar in ('misc:Depends', 'misc:Pre-Depends'): 

202 if substvar not in substvars: 

203 substvars[substvar] = '' 

204 yield substvars 

205 

206 

207def _compute_installed_size(fs_root: FSPath) -> int: 

208 """Emulate dpkg-gencontrol's code for computing the default Installed-Size""" 

209 size_in_kb = 0 

210 hard_links = set() 

211 for path in fs_root.all_paths(): 

212 if not path.is_dir and path.has_fs_path: 

213 st = path.stat() 

214 if st.st_nlink > 1: 

215 hl_key = (st.st_dev, st.st_ino) 

216 if hl_key in hard_links: 

217 continue 

218 hard_links.add(hl_key) 

219 path_size = (st.st_size + 1023) // 1024 

220 else: 

221 path_size = 1 

222 size_in_kb += path_size 

223 return size_in_kb 

224 

225 

226def _generate_dbgsym_control_file_if_relevant(binary_package: BinaryPackage, 

227 dbgsym_root_dir: str, 

228 substvars_file: str, 

229 dbgsym_ids: str, 

230 ) -> None: 

231 section = binary_package.archive_section 

232 component = '' 

233 extra_params = [] 

234 migration = dhe_read_dbgsym_migration(binary_package) 

235 if section is not None and '/' in section and not section.startswith('main/'): 

236 component = section.split('/', 1)[1] + '/' 

237 if binary_package.fields.get('Multi-Arch') != 'same': 

238 extra_params.append('-UMulti-Arch') 

239 if migration: 

240 extra_params.append(f"-DReplaces={migration}") 

241 extra_params.append(f"-DBreaks={migration}") 

242 else: 

243 extra_params.append("-UReplaces") 

244 extra_params.append("-UBreaks") 

245 

246 ensure_dir(os.path.join(dbgsym_root_dir, 'DEBIAN')) 

247 package = binary_package.name 

248 dpkg_cmd = [ 

249 'dpkg-gencontrol', 

250 f'-p{package}', 

251 '-ldebian/changelog', 

252 f'-T{substvars_file}', 

253 f'-P{dbgsym_root_dir}', 

254 f'-DPackage={package}-dbgsym', 

255 '-DDepends=' + package + ' (= ${binary:Version})', 

256 f'-DDescription=debug symbols for {package}', 

257 f'-DSection={component}debug', 

258 f'-DBuild-Ids={dbgsym_ids}', 

259 '-UPre-Depends', 

260 '-URecommends', 

261 '-USuggests', 

262 '-UEnhances', 

263 '-UProvides', 

264 '-UEssential', 

265 '-UConflicts', 

266 '-DPriority=optional', 

267 '-UHomepage', 

268 '-UImportant', 

269 '-UBuilt-Using', 

270 '-DAuto-Built-Package=debug-symbols', 

271 '-UProtected', 

272 *extra_params, 

273 

274 ] 

275 print(f'Executing for {package}-dbgsym: {dpkg_cmd}') 

276 subprocess.check_call(dpkg_cmd) 

277 os.chmod(os.path.join(dbgsym_root_dir, 'DEBIAN', 'control'), 0o644) 

278 

279 

280def _all_parent_directories_of(directories: Iterable[str]) -> Set[str]: 

281 result = {'.'} 

282 for path in directories: 

283 current = os.path.dirname(path) 

284 while current and current not in result: 

285 result.add(current) 

286 current = os.path.dirname(current) 

287 return result 

288 

289 

290def _auto_compute_multi_arch(binary_package: BinaryPackage, 

291 package_root_dir: str, 

292 fs_root: FSPath, 

293 ) -> Optional[str]: 

294 arch = binary_package.fields.get('Architecture') 

295 if arch is None or arch == 'all': 

296 return None 

297 ctrl_dir = os.path.join(package_root_dir, 'DEBIAN') 

298 if any(script for script in ALL_CONTROL_SCRIPTS if os.path.isfile(os.path.join(ctrl_dir, script))): 

299 return None 

300 

301 arch_table = dpkg_architecture_table() 

302 host_multiarch = arch_table.current_host_multiarch 

303 host_arch = arch_table.current_host_arch 

304 acceptable_no_descend_paths = { 

305 f'./usr/lib/{host_multiarch}', 

306 f'./usr/include/{host_multiarch}', 

307 } 

308 acceptable_files = { 

309 f'./usr/share/doc/{binary_package.name}/{basename}' 

310 for basename in ( 

311 'copyright', 

312 'changelog.gz', 

313 'changelog.Debian.gz', 

314 f'changelog.Debian.{host_arch}.gz' 

315 'NEWS.Debian', 

316 'NEWS.Debian.gz', 

317 'README.Debian', 

318 'README.Debian.gz', 

319 ) 

320 } 

321 acceptable_intermediate_dirs = _all_parent_directories_of(itertools.chain(acceptable_no_descend_paths, 

322 acceptable_files)) 

323 

324 for fs_path, children in fs_root.walk(): 

325 path = fs_path.path 

326 if path in acceptable_no_descend_paths: 

327 children.clear() 

328 continue 

329 if path in acceptable_intermediate_dirs or path in acceptable_files: 

330 continue 

331 return None 

332 

333 return 'same' 

334 

335 

336def _generate_control_files(binary_package: BinaryPackage, 

337 package_state: PackageTransformationDefinition, 

338 package_root_dir: str, 

339 fs_root: FSPath, 

340 ) -> None: 

341 package = binary_package.name 

342 substvars_file = f'debian/{package}.substvars' 

343 extra_params = [] 

344 with _ensure_base_substvars_defined(substvars_file) as substvars: 

345 if 'Installed-Size' not in substvars: 

346 # Pass it via cmd-line to make it more visible that we are providing the 

347 # value. It makes it more visible and prevents the dbgsym package from picking it up 

348 extra_params.append(f'-VInstalled-Size={_compute_installed_size(fs_root)}') 

349 

350 if 'Multi-Arch' not in binary_package.fields and not binary_package.is_udeb: 

351 ma_value = _auto_compute_multi_arch(binary_package, package_root_dir, fs_root) 

352 if ma_value is not None: 

353 print(f'The package "{binary_package.name}" looks like it should be "Multi-Arch: {ma_value}" based' 

354 ' on the contents and there is no explicit "Multi-Arch" field. Setting the Multi-Arch field' 

355 ' accordingly in the binary. If this auto-correction is wrong, please add "Multi-Arch: no" to the' 

356 ' relevant part of "debian/control" to disable this feature.') 

357 extra_params.append(f'-DMulti-Arch={ma_value}') 

358 elif binary_package.fields['Multi-Arch'] == 'no': 

359 extra_params.append(f'-UMulti-Arch') 

360 

361 dbgsym_root_dir = dhe_dbgsym_root_dir(binary_package) 

362 dbgsym_ids = dhe_read_dbgsym_build_ids(binary_package) 

363 if os.path.isdir(dbgsym_root_dir): 

364 _generate_dbgsym_control_file_if_relevant(binary_package, dbgsym_root_dir, substvars_file, dbgsym_ids) 

365 elif dbgsym_ids: 

366 extra_params.append(f'-DBuild-Ids={dbgsym_ids}') 

367 

368 if package_state.binary_version is not None: 

369 extra_params.append(f'-v{package_state.binary_version}') 

370 

371 dpkg_cmd = [ 

372 'dpkg-gencontrol', 

373 f'-p{package}', 

374 '-ldebian/changelog', 

375 f'-T{substvars_file}', 

376 f'-P{package_root_dir}', 

377 *extra_params, 

378 ] 

379 print(f'Executing for {package}: {dpkg_cmd}') 

380 subprocess.check_call(dpkg_cmd) 

381 os.chmod(os.path.join(package_root_dir, 'DEBIAN', 'control'), 0o644)