Coverage for deb_packer.py: 64%
188 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1#!/usr/bin/python3 -B
2# setup PYTHONPATH: add our installation directory.
3import pathlib, sys; sys.path.insert(0, pathlib.Path(__file__).parent)
5import argparse
6import io
7import operator
8import os
9import subprocess
10import tarfile
11import textwrap
12import time
14from debian.deb822 import Deb822
16from debputy.intermediate_manifest import TarMember, PathType
17from debputy.util import _error
19try:
20 from typing import Union, NoReturn, Optional, List, FrozenSet, Iterable, IO
21except ImportError:
22 pass
24# AR header / start of a deb file for reference
25# 00000000 21 3c 61 72 63 68 3e 0a 64 65 62 69 61 6e 2d 62 |!<arch>.debian-b|
26# 00000010 69 6e 61 72 79 20 20 20 31 36 36 38 39 37 33 36 |inary 16689736|
27# 00000020 39 35 20 20 30 20 20 20 20 20 30 20 20 20 20 20 |95 0 0 |
28# 00000030 31 30 30 36 34 34 20 20 34 20 20 20 20 20 20 20 |100644 4 |
29# 00000040 20 20 60 0a 32 2e 30 0a 63 6f 6e 74 72 6f 6c 2e | `.2.0.control.|
30# 00000050 74 61 72 2e 78 7a 20 20 31 36 36 38 39 37 33 36 |tar.xz 16689736|
31# 00000060 39 35 20 20 30 20 20 20 20 20 30 20 20 20 20 20 |95 0 0 |
32# 00000070 31 30 30 36 34 34 20 20 39 33 36 38 20 20 20 20 |100644 9368 |
33# 00000080 20 20 60 0a fd 37 7a 58 5a 00 00 04 e6 d6 b4 46 | `..7zXZ......F|
36class ArMember:
38 def __init__(self, name, mtime, fixed_binary=None, write_to_impl=None):
39 self.name = name
40 self._mtime = mtime
41 self._write_to_impl = write_to_impl
42 self.fixed_binary = fixed_binary
44 @property
45 def is_fixed_binary(self):
46 return self.fixed_binary is not None
48 @property
49 def mtime(self):
50 return self.mtime
52 def write_to(self, fd):
53 self._write_to_impl(fd)
56AR_HEADER_LEN = 60
57AR_HEADER = b' ' * AR_HEADER_LEN
60def write_header(fd: IO[bytes], member: ArMember, member_len: int, mtime: int) -> None:
61 header = b'%-16s%-12d0 0 100644 %-10d\x60\n' % (member.name.encode('ascii'), mtime, member_len)
62 fd.write(header)
65def generate_ar_archive(output_filename: str, mtime: int, members: Iterable[ArMember]):
66 with open(output_filename, 'wb', buffering=0) as fd:
67 fd.write(b'!<arch>\n')
68 for member in members:
69 if member.is_fixed_binary:
70 write_header(fd, member, len(member.fixed_binary), mtime)
71 fd.write(member.fixed_binary)
72 else:
73 header_pos = fd.tell()
74 fd.write(AR_HEADER)
75 member.write_to(fd)
76 current_pos = fd.tell()
77 fd.seek(header_pos, os.SEEK_SET)
78 content_len = current_pos - header_pos - AR_HEADER_LEN
79 assert content_len >= 0
80 write_header(fd, member, content_len, mtime)
81 fd.seek(current_pos, os.SEEK_SET)
82 print(f"Generated {output_filename}")
85def _generate_tar_file(tar_members: List[TarMember], compression_cmd, write_to: io.BufferedWriter):
86 with (
87 subprocess.Popen(compression_cmd, stdin=subprocess.PIPE, stdout=write_to) as compress_proc,
88 tarfile.open(mode='w|', fileobj=compress_proc.stdin, format=tarfile.GNU_FORMAT, errorlevel=1) as tar_fd,
89 ):
90 for tar_member in tar_members:
91 tar_info: tarfile.TarInfo = tar_member.create_tar_info(tar_fd)
92 if tar_member.path_type == PathType.FILE:
93 with open(tar_member.fs_path, 'rb') as mfd:
94 tar_fd.addfile(tar_info, fileobj=mfd)
95 else:
96 tar_fd.addfile(tar_info)
97 compress_proc.wait()
98 if compress_proc.returncode != 0: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true
99 _error(f"Compression command {compression_cmd} failed with code {compress_proc.returncode}")
102def generate_tar_file_member(tar_members, compression_cmd):
103 def _impl(fd):
104 return _generate_tar_file(tar_members,
105 compression_cmd,
106 fd,
107 )
108 return _impl
111def _xz_cmdline(compression_rule: 'Compression', parsed_args):
112 compression_level = compression_rule.effective_compression_level(parsed_args)
113 cmdline = ['xz', '-T2', '-' + str(compression_level)]
114 strategy = None if parsed_args is None else parsed_args.compression_strategy
115 if strategy is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false
116 strategy = 'extreme' if parsed_args.is_udeb else 'none'
117 if strategy != "none": 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true
118 cmdline.append('-S' + strategy)
119 return cmdline
122def _gzip_cmdline(compression_rule: 'Compression', parsed_args):
123 compression_level = compression_rule.effective_compression_level(parsed_args)
124 cmdline = ['gzip', '-n' + str(compression_level)]
125 strategy = None if parsed_args is None else parsed_args.compression_strategy
126 if strategy is not None and strategy != 'none':
127 raise ValueError(f'Not implemented: Compression strategy {strategy}'
128 ' for gzip is currently unsupported (but dpkg-deb does)')
129 return cmdline
132def _uncompressed_cmdline(_unused_a: 'Compression', _unused_b):
133 return ['cat']
136class Compression:
138 def __init__(self,
139 default_compression_level: int,
140 extension: str,
141 allowed_strategies: FrozenSet[str],
142 cmdline_builder
143 ):
144 self.default_compression_level = default_compression_level
145 self.extension = extension
146 self.allowed_strategies = allowed_strategies
147 self.cmdline_builder = cmdline_builder
149 def __repr__(self) -> str:
150 return f'<{self.__class__.__name__} {self.extension}>'
152 def effective_compression_level(self, parsed_args) -> int:
153 if parsed_args and parsed_args.compression_level is not None: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 return parsed_args.compression_level
155 return self.default_compression_level
157 def as_cmdline(self, parsed_args) -> List[str]:
158 return self.cmdline_builder(self, parsed_args)
160 def with_extension(self, filename: str) -> str:
161 return filename + self.extension
164COMPRESSIONS = {
165 'xz': Compression(6, '.xz', frozenset({'none', 'extreme'}), _xz_cmdline),
166 'gzip': Compression(9, '.gz', frozenset({'none', 'filtered', 'huffman', 'rle', 'fixed'}), _gzip_cmdline),
167 'none': Compression(0, '', frozenset({'none'}), _uncompressed_cmdline),
168}
171def _normalize_compression_args(parsed_args):
172 if parsed_args.compression_level == 0 and parsed_args.compression_algorithm == 'gzip':
173 print("Note: Mapping compression algorithm to none for compatibility with dpkg-deb (due to -Zgzip -z0)")
174 setattr(parsed_args, 'compression_algorithm', 'none')
176 compression = COMPRESSIONS[parsed_args.compression_algorithm]
177 strategy = parsed_args.compression_strategy
178 if strategy is not None and strategy not in compression.allowed_strategies:
179 _error(f'Compression algorithm "{parsed_args.compression_algorithm}" does not support compression strategy'
180 f' "{strategy}". Allowed values: {", ".join(sorted(compression.allowed_strategies))}')
181 return parsed_args
184def _compute_output_filename(package_root_dir: str, is_udeb: bool) -> str:
185 with open(os.path.join(package_root_dir, 'DEBIAN', 'control'), 'rt') as fd:
186 control_file = Deb822(fd)
188 package_name = control_file['Package']
189 package_version = control_file['Version']
190 package_architecture = control_file['Architecture']
191 extension = control_file.get('Package-Type')
192 if ':' in package_version:
193 package_version = package_version.split(':', 1)[1]
194 if extension is None: 194 ↛ 196line 194 didn't jump to line 196, because the condition on line 194 was never false
195 extension = 'deb'
196 if is_udeb:
197 extension = 'udeb'
199 return f'{package_name}_{package_version}_{package_architecture}.{extension}'
202def parse_args():
203 description = textwrap.dedent(f'''\
204 THIS IS A PROTOTYPE "dpkg-deb -b" emulator with basic manifest support
206 DO NOT USE THIS TOOL DIRECTLY. It has not stability guarantees and will be removed as
207 soon as "dpkg-deb -b" grows support for the relevant features.
209 This tool is a prototype "dpkg-deb -b"-like interface for compiling a Debian package
210 without requiring root even for static ownership. It is a temporary stand-in for
211 "dpkg-deb -b" until "dpkg-deb -b" will get support for a manifest.
213 The tool operates on an internal JSON based manifest for now, because it was faster
214 than building an mtree parser (which is the format that dpkg will likely end up
215 using).
217 As the tool is not meant to be used directly, it is full of annoying paper cuts that
218 I refuse to fix or maintain. Use the high level tool instead.
220 ''')
222 parser = argparse.ArgumentParser(
223 description=description,
224 formatter_class=argparse.RawDescriptionHelpFormatter,
225 )
226 parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR",
227 help='Root directory of the package. Must contain a DEBIAN directory'
228 )
229 parser.add_argument('package_output_path', metavar="PATH",
230 help='Path where the package should be placed. If it is directory,'
231 ' the base name will be determined from the package metadata'
232 )
234 parser.add_argument('--udeb', dest='is_udeb',
235 action='store_true', default=False,
236 help='The package being built is a udeb.'
237 )
239 parser.add_argument('--intermediate-package-manifest', dest='package_manifest', metavar="JSON_FILE",
240 action='store', default=None,
241 help='INTERMEDIATE package manifest (JSON!)')
242 parser.add_argument('--root-owner-group', dest='root_owner_group', action='store_true',
243 help='Ignored. Accepted for compatibility with dpkg-deb -b')
244 parser.add_argument('-b', '--build', dest='build_param', action='store_true',
245 help='Ignored. Accepted for compatibility with dpkg-deb')
246 parser.add_argument('--source-date-epoch', dest='source_date_epoch',
247 action='store', type=int, default=None,
248 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable'
249 )
250 parser.add_argument('-Z', dest='compression_algorithm', choices=COMPRESSIONS, default='xz',
251 help='The compression algorithm to be used'
252 )
253 parser.add_argument('-z', dest='compression_level', metavar='{0-9}',
254 choices=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
255 default=None,
256 type=int, help='The compression level to be used'
257 )
258 parser.add_argument('-S', dest='compression_strategy',
259 # We have a different default for xz when strategy is unset and we are building a udeb
260 action="store", default=None,
261 help='The compression algorithm to be used. Concrete values depend on the compression'
262 ' algorithm, but the value "none" is always allowed'
263 )
264 parser.add_argument('--uniform-compression', dest='uniform_compression',
265 action="store_true", default=True,
266 help='Whether to use the same compression for the control.tar and the data.tar.'
267 ' The default is to use uniform compression.'
269 )
270 parser.add_argument('--no-uniform-compression', dest='uniform_compression',
271 action="store_false", default=True,
272 help='Disable uniform compression (see --uniform-compression)'
273 )
275 parsed_args = parser.parse_args()
276 parsed_args = _normalize_compression_args(parsed_args)
278 return parsed_args
281def _ctrl_member(member_path, fs_path=None, path_type=PathType.FILE, mode=0o644, mtime=0):
282 if fs_path is None: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true
283 assert member_path.startswith('./')
284 fs_path = 'DEBIAN' + member_path[1:]
285 return TarMember(
286 member_path=member_path,
287 path_type=path_type,
288 fs_path=fs_path,
289 mode=mode,
290 owner='root',
291 uid=0,
292 group='root',
293 gid=0,
294 mtime=mtime,
295 )
298CTRL_MEMBER_SCRIPTS = {'postinst', 'preinst', 'postrm', 'prerm', 'config', 'isinstallable'}
301def _ctrl_tar_members(package_root_dir: str, mtime: int) -> Iterable[TarMember]:
302 debian_root = os.path.join(package_root_dir, 'DEBIAN')
303 yield _ctrl_member('./', debian_root, path_type=PathType.DIRECTORY, mode=0o0755, mtime=mtime)
304 with os.scandir(debian_root) as dir_iter:
305 for ctrl_member in sorted(dir_iter, key=operator.attrgetter('name')):
306 if not ctrl_member.is_file: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true
307 _error(f"{ctrl_member.path} is not a file and all control.tar members ought to be files!")
308 yield _ctrl_member(
309 f'./{ctrl_member.name}',
310 path_type=PathType.FILE,
311 fs_path=ctrl_member.path,
312 mode=0o0755 if ctrl_member.name in CTRL_MEMBER_SCRIPTS else 0o0644,
313 mtime=mtime,
314 )
317def parse_manifest(manifest_path: 'Optional[str]') -> 'List[TarMember]':
318 if manifest_path is None: 318 ↛ 319line 318 didn't jump to line 319, because the condition on line 318 was never true
319 _error(f'--intermediate-package-manifest is mandatory for now')
320 return TarMember.parse_intermediate_manifest(manifest_path)
323def main():
324 parsed_args = parse_args()
325 root_dir: str = parsed_args.package_root_dir
326 output_path: str = parsed_args.package_output_path
328 mtime: int = parsed_args.source_date_epoch
329 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ:
330 sde_raw = os.environ['SOURCE_DATE_EPOCH']
331 if sde_raw == '':
332 # If you replay the example from bash history, but it is the first run in the session, and you did not
333 # run the "stat" line.
334 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the"
335 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line")
336 mtime = int(sde_raw)
337 if mtime is None:
338 mtime = int(time.time())
340 data_compression: Compression = COMPRESSIONS[parsed_args.compression_algorithm]
341 data_compression_cmd = data_compression.as_cmdline(parsed_args)
342 if parsed_args.uniform_compression:
343 ctrl_compression = data_compression
344 ctrl_compression_cmd = data_compression_cmd
345 else:
346 ctrl_compression = COMPRESSIONS['gzip']
347 ctrl_compression_cmd = COMPRESSIONS['gzip'].as_cmdline(None)
349 if output_path.endswith('/') or os.path.isdir(output_path):
350 deb_file = os.path.join(output_path, _compute_output_filename(root_dir, parsed_args.is_udeb))
351 else:
352 deb_file = output_path
354 pack(deb_file, ctrl_compression, data_compression, root_dir, parsed_args.package_manifest,
355 mtime, ctrl_compression_cmd, data_compression_cmd)
358def pack(deb_file: str, ctrl_compression: Compression, data_compression: Compression, root_dir: str,
359 package_manifest: 'Optional[str]', mtime: int, ctrl_compression_cmd, data_compression_cmd):
361 data_tar_members = parse_manifest(package_manifest)
362 members = [
363 ArMember('debian-binary', mtime, fixed_binary=b'2.0\n'),
364 ArMember(ctrl_compression.with_extension('control.tar'), mtime,
365 write_to_impl=generate_tar_file_member(
366 _ctrl_tar_members(root_dir, mtime),
367 ctrl_compression_cmd
368 )),
369 ArMember(data_compression.with_extension('data.tar'), mtime,
370 write_to_impl=generate_tar_file_member(
371 data_tar_members,
372 data_compression_cmd
373 )),
374 ]
375 generate_ar_archive(deb_file, mtime, members)
378if __name__ == '__main__':
379 main()