Coverage for debputy/deb_packaging_support.py: 0%
216 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1import contextlib
2import hashlib
3import itertools
4import os
5import shutil
6import subprocess
7from typing import Iterable, List, TextIO, Optional, Literal, Set, Any, Generator
9from debian.substvars import Substvars
11from debputy.architecture_support import dpkg_architecture_table
12from debputy.debhelper_emulation import dhe_install_pkg_file_as_ctrl_file_if_present, dhe_generated_file, \
13 dhe_dbgsym_root_dir, dhe_read_dbgsym_build_ids, dhe_read_dbgsym_migration
14from debputy.filesystem_scan import FSPath
15from debputy.highlevel_manifest import HighLevelManifest, PackageTransformationDefinition
16from debputy.installdeb_emulation import dh_installdeb_emulation
17from debputy.maintscript_snippet import UDEB_CONTROL_SCRIPTS, MaintscriptSnippet, STD_CONTROL_SCRIPTS, \
18 ALL_CONTROL_SCRIPTS
19from debputy.packages import BinaryPackage
20from debputy.util import debputy_autoscript_tagline, _error, ensure_dir
23def generate_md5sums_file(root_dir: str, fs_root: FSPath) -> None:
24 conffiles = os.path.join(root_dir, 'DEBIAN', 'conffiles')
25 md5sums = os.path.join(root_dir, 'DEBIAN', 'md5sums')
26 exclude = set()
27 if os.path.isfile(conffiles):
28 with open(conffiles, 'rt') as fd:
29 for line in fd:
30 if not line.startswith("/"):
31 continue
32 exclude.add('.' + line.rstrip("\n"))
33 had_content = False
34 files = (path for path in fs_root.all_paths() if path.is_file and path.path not in exclude)
35 with open(md5sums, 'wt') as md5fd:
36 for member in files:
37 path = member.path
38 assert path.startswith('./')
39 path = path[2:]
40 with open(member.fs_path, 'rb') as f:
41 file_hash = hashlib.md5()
42 while chunk := f.read(8192):
43 file_hash.update(chunk)
44 had_content = True
45 md5fd.write(f"{file_hash.hexdigest()} {path}\n")
46 if not had_content:
47 os.unlink(md5sums)
50def install_or_generate_conffiles(binary_package: BinaryPackage, root_dir: str, fs_root: FSPath) -> None:
51 conffiles_dest = os.path.join(root_dir, 'DEBIAN', 'conffiles')
52 dhe_install_pkg_file_as_ctrl_file_if_present(binary_package, 'conffiles', root_dir, 0o0644)
53 etc_dir = fs_root.lookup('etc')
54 if etc_dir:
55 _add_conffiles(conffiles_dest, (p for p in etc_dir.all_paths() if p.is_file))
56 if os.path.isfile(conffiles_dest):
57 os.chmod(conffiles_dest, 0o0644)
60def build_dbgsym_package_if_relevant(binary_package: BinaryPackage,
61 dpkg_deb_args: List[str],
62 output_dir: str,
63 ) -> None:
64 if binary_package.is_udeb:
65 # We never built udebs due to #797391. If it is easy to do, we can fix it when we do the control file
66 return
67 if not os.path.isdir(output_dir):
68 _error(f"Cannot produce a dbgsym package when output path is not a directory.")
69 return
70 dbgsym_root = dhe_dbgsym_root_dir(binary_package)
71 if not os.path.isdir(dbgsym_root):
72 return
73 package = binary_package.name
74 if not os.path.isfile(os.path.join(dbgsym_root, 'DEBIAN', 'control')):
75 # Will not be a problem when we integrate generation of DEBIAN/control (#18)
76 _error(f"There is a dbgsym directory from debhelper for {package}, but not a DEBIAN/control")
78 dpkg_cmd = [
79 'dpkg-deb',
80 '--root-owner-group',
81 ]
82 dpkg_cmd.extend(dpkg_deb_args)
83 dpkg_cmd.extend([
84 '--build',
85 dbgsym_root,
86 output_dir,
87 ])
89 print(f"Executing for {package}-dbgsym: {dpkg_cmd}")
90 subprocess.check_call(dpkg_cmd)
93def setup_control_files(binary_package: BinaryPackage,
94 root_dir: str,
95 fs_root: FSPath,
96 manifest: HighLevelManifest,
97 ) -> None:
98 ensure_dir(os.path.join(root_dir, 'DEBIAN'))
99 package_state = manifest.package_state_for(binary_package.name)
100 if binary_package.is_udeb:
101 _generate_control_files(binary_package, package_state, root_dir, fs_root)
102 for script in UDEB_CONTROL_SCRIPTS:
103 dhe_install_pkg_file_as_ctrl_file_if_present(binary_package, script, root_dir, 0o0755)
104 return
106 state = manifest.package_state_for(binary_package.name)
107 if state.maintscript_snippets:
108 for script, snippets in state.maintscript_snippets.items():
109 _insert_autoscript_snippets(
110 binary_package,
111 script,
112 snippets,
113 debputy_autoscript_tagline(),
114 )
116 dh_installdeb_emulation(binary_package, root_dir)
117 install_or_generate_conffiles(binary_package, root_dir, fs_root)
118 generate_md5sums_file(root_dir, fs_root)
119 _generate_control_files(binary_package, package_state, root_dir, fs_root)
122def _add_conffiles(conffiles_dest: str, conffile_matches: Iterable[FSPath]) -> None:
123 with open(conffiles_dest, 'at') as fd:
124 for conffile_match in conffile_matches:
125 conffile = conffile_match.path
126 assert conffile_match.is_file
127 assert conffile.startswith('./')
128 fd.write(f"{conffile[1:]}\n")
129 if os.stat(conffiles_dest).st_size == 0:
130 os.utime(conffiles_dest)
133def _add_snippets_to_script(fd: TextIO,
134 snippets: Iterable[MaintscriptSnippet],
135 tool_with_version: str,
136 ) -> None:
138 fd.write(f"# Automatically added by {tool_with_version}\n")
139 for snippet in snippets:
140 fd.write(f'# - source: {snippet.definition_source}\n')
141 fd.write(snippet.snippet)
142 if not snippet.snippet.endswith('\n'):
143 fd.write('\n')
144 fd.write("# End automatically added section\n")
147def _insert_by_snippet_type(binary_package: BinaryPackage,
148 script: str,
149 snippet_order: Optional[Literal['service']],
150 snippets: List[MaintscriptSnippet],
151 reverse_order: bool,
152 tool_with_version: str,
153 ) -> None:
154 if snippet_order is None:
155 outfile = f'debian/{binary_package.name}.{script}.debhelper'
156 else:
157 outfile = dhe_generated_file(binary_package, f"{script}.{snippet_order}")
159 if reverse_order:
160 with open(outfile, 'wt') as wfd, open(outfile, 'rt') as rfd:
161 _add_snippets_to_script(wfd, reversed(snippets), tool_with_version)
162 shutil.copyfileobj(rfd, wfd)
163 else:
164 with open(outfile, 'at') as fd:
165 _add_snippets_to_script(fd, snippets, tool_with_version)
168def _insert_autoscript_snippets(binary_package: BinaryPackage,
169 script: str,
170 snippets: List[MaintscriptSnippet],
171 tool_with_version: str,
172 ) -> None:
174 assert script in STD_CONTROL_SCRIPTS
175 reverse_order = script in ('postrm', 'prerm')
176 general_snippets = [s for s in snippets if s.snippet_order is None]
177 service_snippets = [s for s in snippets if s.snippet_order == 'service']
178 if general_snippets:
179 _insert_by_snippet_type(
180 binary_package,
181 script,
182 None,
183 snippets,
184 reverse_order,
185 tool_with_version
186 )
187 if service_snippets:
188 _insert_by_snippet_type(
189 binary_package,
190 script,
191 'service',
192 snippets,
193 reverse_order,
194 tool_with_version
195 )
198@contextlib.contextmanager
199def _ensure_base_substvars_defined(substvars_file: str) -> Generator[Substvars, Any, Any]:
200 with Substvars.load_from_path(substvars_file, missing_ok=True) as substvars:
201 for substvar in ('misc:Depends', 'misc:Pre-Depends'):
202 if substvar not in substvars:
203 substvars[substvar] = ''
204 yield substvars
207def _compute_installed_size(fs_root: FSPath) -> int:
208 """Emulate dpkg-gencontrol's code for computing the default Installed-Size"""
209 size_in_kb = 0
210 hard_links = set()
211 for path in fs_root.all_paths():
212 if not path.is_dir and path.has_fs_path:
213 st = path.stat()
214 if st.st_nlink > 1:
215 hl_key = (st.st_dev, st.st_ino)
216 if hl_key in hard_links:
217 continue
218 hard_links.add(hl_key)
219 path_size = (st.st_size + 1023) // 1024
220 else:
221 path_size = 1
222 size_in_kb += path_size
223 return size_in_kb
226def _generate_dbgsym_control_file_if_relevant(binary_package: BinaryPackage,
227 dbgsym_root_dir: str,
228 substvars_file: str,
229 dbgsym_ids: str,
230 ) -> None:
231 section = binary_package.archive_section
232 component = ''
233 extra_params = []
234 migration = dhe_read_dbgsym_migration(binary_package)
235 if section is not None and '/' in section and not section.startswith('main/'):
236 component = section.split('/', 1)[1] + '/'
237 if binary_package.fields.get('Multi-Arch') != 'same':
238 extra_params.append('-UMulti-Arch')
239 if migration:
240 extra_params.append(f"-DReplaces={migration}")
241 extra_params.append(f"-DBreaks={migration}")
242 else:
243 extra_params.append("-UReplaces")
244 extra_params.append("-UBreaks")
246 ensure_dir(os.path.join(dbgsym_root_dir, 'DEBIAN'))
247 package = binary_package.name
248 dpkg_cmd = [
249 'dpkg-gencontrol',
250 f'-p{package}',
251 '-ldebian/changelog',
252 f'-T{substvars_file}',
253 f'-P{dbgsym_root_dir}',
254 f'-DPackage={package}-dbgsym',
255 '-DDepends=' + package + ' (= ${binary:Version})',
256 f'-DDescription=debug symbols for {package}',
257 f'-DSection={component}debug',
258 f'-DBuild-Ids={dbgsym_ids}',
259 '-UPre-Depends',
260 '-URecommends',
261 '-USuggests',
262 '-UEnhances',
263 '-UProvides',
264 '-UEssential',
265 '-UConflicts',
266 '-DPriority=optional',
267 '-UHomepage',
268 '-UImportant',
269 '-UBuilt-Using',
270 '-DAuto-Built-Package=debug-symbols',
271 '-UProtected',
272 *extra_params,
274 ]
275 print(f'Executing for {package}-dbgsym: {dpkg_cmd}')
276 subprocess.check_call(dpkg_cmd)
277 os.chmod(os.path.join(dbgsym_root_dir, 'DEBIAN', 'control'), 0o644)
280def _all_parent_directories_of(directories: Iterable[str]) -> Set[str]:
281 result = {'.'}
282 for path in directories:
283 current = os.path.dirname(path)
284 while current and current not in result:
285 result.add(current)
286 current = os.path.dirname(current)
287 return result
290def _auto_compute_multi_arch(binary_package: BinaryPackage,
291 package_root_dir: str,
292 fs_root: FSPath,
293 ) -> Optional[str]:
294 arch = binary_package.fields.get('Architecture')
295 if arch is None or arch == 'all':
296 return None
297 ctrl_dir = os.path.join(package_root_dir, 'DEBIAN')
298 if any(script for script in ALL_CONTROL_SCRIPTS if os.path.isfile(os.path.join(ctrl_dir, script))):
299 return None
301 arch_table = dpkg_architecture_table()
302 host_multiarch = arch_table.current_host_multiarch
303 host_arch = arch_table.current_host_arch
304 acceptable_no_descend_paths = {
305 f'./usr/lib/{host_multiarch}',
306 f'./usr/include/{host_multiarch}',
307 }
308 acceptable_files = {
309 f'./usr/share/doc/{binary_package.name}/{basename}'
310 for basename in (
311 'copyright',
312 'changelog.gz',
313 'changelog.Debian.gz',
314 f'changelog.Debian.{host_arch}.gz'
315 'NEWS.Debian',
316 'NEWS.Debian.gz',
317 'README.Debian',
318 'README.Debian.gz',
319 )
320 }
321 acceptable_intermediate_dirs = _all_parent_directories_of(itertools.chain(acceptable_no_descend_paths,
322 acceptable_files))
324 for fs_path, children in fs_root.walk():
325 path = fs_path.path
326 if path in acceptable_no_descend_paths:
327 children.clear()
328 continue
329 if path in acceptable_intermediate_dirs or path in acceptable_files:
330 continue
331 return None
333 return 'same'
336def _generate_control_files(binary_package: BinaryPackage,
337 package_state: PackageTransformationDefinition,
338 package_root_dir: str,
339 fs_root: FSPath,
340 ) -> None:
341 package = binary_package.name
342 substvars_file = f'debian/{package}.substvars'
343 extra_params = []
344 with _ensure_base_substvars_defined(substvars_file) as substvars:
345 if 'Installed-Size' not in substvars:
346 # Pass it via cmd-line to make it more visible that we are providing the
347 # value. It makes it more visible and prevents the dbgsym package from picking it up
348 extra_params.append(f'-VInstalled-Size={_compute_installed_size(fs_root)}')
350 if 'Multi-Arch' not in binary_package.fields and not binary_package.is_udeb:
351 ma_value = _auto_compute_multi_arch(binary_package, package_root_dir, fs_root)
352 if ma_value is not None:
353 print(f'The package "{binary_package.name}" looks like it should be "Multi-Arch: {ma_value}" based'
354 ' on the contents and there is no explicit "Multi-Arch" field. Setting the Multi-Arch field'
355 ' accordingly in the binary. If this auto-correction is wrong, please add "Multi-Arch: no" to the'
356 ' relevant part of "debian/control" to disable this feature.')
357 extra_params.append(f'-DMulti-Arch={ma_value}')
358 elif binary_package.fields['Multi-Arch'] == 'no':
359 extra_params.append(f'-UMulti-Arch')
361 dbgsym_root_dir = dhe_dbgsym_root_dir(binary_package)
362 dbgsym_ids = dhe_read_dbgsym_build_ids(binary_package)
363 if os.path.isdir(dbgsym_root_dir):
364 _generate_dbgsym_control_file_if_relevant(binary_package, dbgsym_root_dir, substvars_file, dbgsym_ids)
365 elif dbgsym_ids:
366 extra_params.append(f'-DBuild-Ids={dbgsym_ids}')
368 if package_state.binary_version is not None:
369 extra_params.append(f'-v{package_state.binary_version}')
371 dpkg_cmd = [
372 'dpkg-gencontrol',
373 f'-p{package}',
374 '-ldebian/changelog',
375 f'-T{substvars_file}',
376 f'-P{package_root_dir}',
377 *extra_params,
378 ]
379 print(f'Executing for {package}: {dpkg_cmd}')
380 subprocess.check_call(dpkg_cmd)
381 os.chmod(os.path.join(package_root_dir, 'DEBIAN', 'control'), 0o644)