Coverage for deb_packer.py: 64%

188 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-22 14:29 +0100

1#!/usr/bin/python3 -B 

2# setup PYTHONPATH: add our installation directory. 

3import pathlib, sys; sys.path.insert(0, pathlib.Path(__file__).parent) 

4 

5import argparse 

6import io 

7import operator 

8import os 

9import subprocess 

10import tarfile 

11import textwrap 

12import time 

13 

14from debian.deb822 import Deb822 

15 

16from debputy.intermediate_manifest import TarMember, PathType 

17from debputy.util import _error 

18 

19try: 

20 from typing import Union, NoReturn, Optional, List, FrozenSet, Iterable, IO 

21except ImportError: 

22 pass 

23 

24# AR header / start of a deb file for reference 

25# 00000000 21 3c 61 72 63 68 3e 0a 64 65 62 69 61 6e 2d 62 |!<arch>.debian-b| 

26# 00000010 69 6e 61 72 79 20 20 20 31 36 36 38 39 37 33 36 |inary 16689736| 

27# 00000020 39 35 20 20 30 20 20 20 20 20 30 20 20 20 20 20 |95 0 0 | 

28# 00000030 31 30 30 36 34 34 20 20 34 20 20 20 20 20 20 20 |100644 4 | 

29# 00000040 20 20 60 0a 32 2e 30 0a 63 6f 6e 74 72 6f 6c 2e | `.2.0.control.| 

30# 00000050 74 61 72 2e 78 7a 20 20 31 36 36 38 39 37 33 36 |tar.xz 16689736| 

31# 00000060 39 35 20 20 30 20 20 20 20 20 30 20 20 20 20 20 |95 0 0 | 

32# 00000070 31 30 30 36 34 34 20 20 39 33 36 38 20 20 20 20 |100644 9368 | 

33# 00000080 20 20 60 0a fd 37 7a 58 5a 00 00 04 e6 d6 b4 46 | `..7zXZ......F| 

34 

35 

36class ArMember: 

37 

38 def __init__(self, name, mtime, fixed_binary=None, write_to_impl=None): 

39 self.name = name 

40 self._mtime = mtime 

41 self._write_to_impl = write_to_impl 

42 self.fixed_binary = fixed_binary 

43 

44 @property 

45 def is_fixed_binary(self): 

46 return self.fixed_binary is not None 

47 

48 @property 

49 def mtime(self): 

50 return self.mtime 

51 

52 def write_to(self, fd): 

53 self._write_to_impl(fd) 

54 

55 

56AR_HEADER_LEN = 60 

57AR_HEADER = b' ' * AR_HEADER_LEN 

58 

59 

60def write_header(fd: IO[bytes], member: ArMember, member_len: int, mtime: int) -> None: 

61 header = b'%-16s%-12d0 0 100644 %-10d\x60\n' % (member.name.encode('ascii'), mtime, member_len) 

62 fd.write(header) 

63 

64 

65def generate_ar_archive(output_filename: str, mtime: int, members: Iterable[ArMember]): 

66 with open(output_filename, 'wb', buffering=0) as fd: 

67 fd.write(b'!<arch>\n') 

68 for member in members: 

69 if member.is_fixed_binary: 

70 write_header(fd, member, len(member.fixed_binary), mtime) 

71 fd.write(member.fixed_binary) 

72 else: 

73 header_pos = fd.tell() 

74 fd.write(AR_HEADER) 

75 member.write_to(fd) 

76 current_pos = fd.tell() 

77 fd.seek(header_pos, os.SEEK_SET) 

78 content_len = current_pos - header_pos - AR_HEADER_LEN 

79 assert content_len >= 0 

80 write_header(fd, member, content_len, mtime) 

81 fd.seek(current_pos, os.SEEK_SET) 

82 print(f"Generated {output_filename}") 

83 

84 

85def _generate_tar_file(tar_members: List[TarMember], compression_cmd, write_to: io.BufferedWriter): 

86 with ( 

87 subprocess.Popen(compression_cmd, stdin=subprocess.PIPE, stdout=write_to) as compress_proc, 

88 tarfile.open(mode='w|', fileobj=compress_proc.stdin, format=tarfile.GNU_FORMAT, errorlevel=1) as tar_fd, 

89 ): 

90 for tar_member in tar_members: 

91 tar_info: tarfile.TarInfo = tar_member.create_tar_info(tar_fd) 

92 if tar_member.path_type == PathType.FILE: 

93 with open(tar_member.fs_path, 'rb') as mfd: 

94 tar_fd.addfile(tar_info, fileobj=mfd) 

95 else: 

96 tar_fd.addfile(tar_info) 

97 compress_proc.wait() 

98 if compress_proc.returncode != 0: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true

99 _error(f"Compression command {compression_cmd} failed with code {compress_proc.returncode}") 

100 

101 

102def generate_tar_file_member(tar_members, compression_cmd): 

103 def _impl(fd): 

104 return _generate_tar_file(tar_members, 

105 compression_cmd, 

106 fd, 

107 ) 

108 return _impl 

109 

110 

111def _xz_cmdline(compression_rule: 'Compression', parsed_args): 

112 compression_level = compression_rule.effective_compression_level(parsed_args) 

113 cmdline = ['xz', '-T2', '-' + str(compression_level)] 

114 strategy = None if parsed_args is None else parsed_args.compression_strategy 

115 if strategy is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false

116 strategy = 'extreme' if parsed_args.is_udeb else 'none' 

117 if strategy != "none": 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true

118 cmdline.append('-S' + strategy) 

119 return cmdline 

120 

121 

122def _gzip_cmdline(compression_rule: 'Compression', parsed_args): 

123 compression_level = compression_rule.effective_compression_level(parsed_args) 

124 cmdline = ['gzip', '-n' + str(compression_level)] 

125 strategy = None if parsed_args is None else parsed_args.compression_strategy 

126 if strategy is not None and strategy != 'none': 

127 raise ValueError(f'Not implemented: Compression strategy {strategy}' 

128 ' for gzip is currently unsupported (but dpkg-deb does)') 

129 return cmdline 

130 

131 

132def _uncompressed_cmdline(_unused_a: 'Compression', _unused_b): 

133 return ['cat'] 

134 

135 

136class Compression: 

137 

138 def __init__(self, 

139 default_compression_level: int, 

140 extension: str, 

141 allowed_strategies: FrozenSet[str], 

142 cmdline_builder 

143 ): 

144 self.default_compression_level = default_compression_level 

145 self.extension = extension 

146 self.allowed_strategies = allowed_strategies 

147 self.cmdline_builder = cmdline_builder 

148 

149 def __repr__(self) -> str: 

150 return f'<{self.__class__.__name__} {self.extension}>' 

151 

152 def effective_compression_level(self, parsed_args) -> int: 

153 if parsed_args and parsed_args.compression_level is not None: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 return parsed_args.compression_level 

155 return self.default_compression_level 

156 

157 def as_cmdline(self, parsed_args) -> List[str]: 

158 return self.cmdline_builder(self, parsed_args) 

159 

160 def with_extension(self, filename: str) -> str: 

161 return filename + self.extension 

162 

163 

164COMPRESSIONS = { 

165 'xz': Compression(6, '.xz', frozenset({'none', 'extreme'}), _xz_cmdline), 

166 'gzip': Compression(9, '.gz', frozenset({'none', 'filtered', 'huffman', 'rle', 'fixed'}), _gzip_cmdline), 

167 'none': Compression(0, '', frozenset({'none'}), _uncompressed_cmdline), 

168} 

169 

170 

171def _normalize_compression_args(parsed_args): 

172 if parsed_args.compression_level == 0 and parsed_args.compression_algorithm == 'gzip': 

173 print("Note: Mapping compression algorithm to none for compatibility with dpkg-deb (due to -Zgzip -z0)") 

174 setattr(parsed_args, 'compression_algorithm', 'none') 

175 

176 compression = COMPRESSIONS[parsed_args.compression_algorithm] 

177 strategy = parsed_args.compression_strategy 

178 if strategy is not None and strategy not in compression.allowed_strategies: 

179 _error(f'Compression algorithm "{parsed_args.compression_algorithm}" does not support compression strategy' 

180 f' "{strategy}". Allowed values: {", ".join(sorted(compression.allowed_strategies))}') 

181 return parsed_args 

182 

183 

184def _compute_output_filename(package_root_dir: str, is_udeb: bool) -> str: 

185 with open(os.path.join(package_root_dir, 'DEBIAN', 'control'), 'rt') as fd: 

186 control_file = Deb822(fd) 

187 

188 package_name = control_file['Package'] 

189 package_version = control_file['Version'] 

190 package_architecture = control_file['Architecture'] 

191 extension = control_file.get('Package-Type') 

192 if ':' in package_version: 

193 package_version = package_version.split(':', 1)[1] 

194 if extension is None: 194 ↛ 196line 194 didn't jump to line 196, because the condition on line 194 was never false

195 extension = 'deb' 

196 if is_udeb: 

197 extension = 'udeb' 

198 

199 return f'{package_name}_{package_version}_{package_architecture}.{extension}' 

200 

201 

202def parse_args(): 

203 description = textwrap.dedent(f'''\ 

204 THIS IS A PROTOTYPE "dpkg-deb -b" emulator with basic manifest support 

205 

206 DO NOT USE THIS TOOL DIRECTLY. It has not stability guarantees and will be removed as 

207 soon as "dpkg-deb -b" grows support for the relevant features.  

208 

209 This tool is a prototype "dpkg-deb -b"-like interface for compiling a Debian package 

210 without requiring root even for static ownership. It is a temporary stand-in for 

211 "dpkg-deb -b" until "dpkg-deb -b" will get support for a manifest. 

212 

213 The tool operates on an internal JSON based manifest for now, because it was faster 

214 than building an mtree parser (which is the format that dpkg will likely end up 

215 using). 

216  

217 As the tool is not meant to be used directly, it is full of annoying paper cuts that 

218 I refuse to fix or maintain. Use the high level tool instead. 

219 

220 ''') 

221 

222 parser = argparse.ArgumentParser( 

223 description=description, 

224 formatter_class=argparse.RawDescriptionHelpFormatter, 

225 ) 

226 parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR", 

227 help='Root directory of the package. Must contain a DEBIAN directory' 

228 ) 

229 parser.add_argument('package_output_path', metavar="PATH", 

230 help='Path where the package should be placed. If it is directory,' 

231 ' the base name will be determined from the package metadata' 

232 ) 

233 

234 parser.add_argument('--udeb', dest='is_udeb', 

235 action='store_true', default=False, 

236 help='The package being built is a udeb.' 

237 ) 

238 

239 parser.add_argument('--intermediate-package-manifest', dest='package_manifest', metavar="JSON_FILE", 

240 action='store', default=None, 

241 help='INTERMEDIATE package manifest (JSON!)') 

242 parser.add_argument('--root-owner-group', dest='root_owner_group', action='store_true', 

243 help='Ignored. Accepted for compatibility with dpkg-deb -b') 

244 parser.add_argument('-b', '--build', dest='build_param', action='store_true', 

245 help='Ignored. Accepted for compatibility with dpkg-deb') 

246 parser.add_argument('--source-date-epoch', dest='source_date_epoch', 

247 action='store', type=int, default=None, 

248 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable' 

249 ) 

250 parser.add_argument('-Z', dest='compression_algorithm', choices=COMPRESSIONS, default='xz', 

251 help='The compression algorithm to be used' 

252 ) 

253 parser.add_argument('-z', dest='compression_level', metavar='{0-9}', 

254 choices=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 

255 default=None, 

256 type=int, help='The compression level to be used' 

257 ) 

258 parser.add_argument('-S', dest='compression_strategy', 

259 # We have a different default for xz when strategy is unset and we are building a udeb 

260 action="store", default=None, 

261 help='The compression algorithm to be used. Concrete values depend on the compression' 

262 ' algorithm, but the value "none" is always allowed' 

263 ) 

264 parser.add_argument('--uniform-compression', dest='uniform_compression', 

265 action="store_true", default=True, 

266 help='Whether to use the same compression for the control.tar and the data.tar.' 

267 ' The default is to use uniform compression.' 

268 

269 ) 

270 parser.add_argument('--no-uniform-compression', dest='uniform_compression', 

271 action="store_false", default=True, 

272 help='Disable uniform compression (see --uniform-compression)' 

273 ) 

274 

275 parsed_args = parser.parse_args() 

276 parsed_args = _normalize_compression_args(parsed_args) 

277 

278 return parsed_args 

279 

280 

281def _ctrl_member(member_path, fs_path=None, path_type=PathType.FILE, mode=0o644, mtime=0): 

282 if fs_path is None: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 assert member_path.startswith('./') 

284 fs_path = 'DEBIAN' + member_path[1:] 

285 return TarMember( 

286 member_path=member_path, 

287 path_type=path_type, 

288 fs_path=fs_path, 

289 mode=mode, 

290 owner='root', 

291 uid=0, 

292 group='root', 

293 gid=0, 

294 mtime=mtime, 

295 ) 

296 

297 

298CTRL_MEMBER_SCRIPTS = {'postinst', 'preinst', 'postrm', 'prerm', 'config', 'isinstallable'} 

299 

300 

301def _ctrl_tar_members(package_root_dir: str, mtime: int) -> Iterable[TarMember]: 

302 debian_root = os.path.join(package_root_dir, 'DEBIAN') 

303 yield _ctrl_member('./', debian_root, path_type=PathType.DIRECTORY, mode=0o0755, mtime=mtime) 

304 with os.scandir(debian_root) as dir_iter: 

305 for ctrl_member in sorted(dir_iter, key=operator.attrgetter('name')): 

306 if not ctrl_member.is_file: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true

307 _error(f"{ctrl_member.path} is not a file and all control.tar members ought to be files!") 

308 yield _ctrl_member( 

309 f'./{ctrl_member.name}', 

310 path_type=PathType.FILE, 

311 fs_path=ctrl_member.path, 

312 mode=0o0755 if ctrl_member.name in CTRL_MEMBER_SCRIPTS else 0o0644, 

313 mtime=mtime, 

314 ) 

315 

316 

317def parse_manifest(manifest_path: 'Optional[str]') -> 'List[TarMember]': 

318 if manifest_path is None: 318 ↛ 319line 318 didn't jump to line 319, because the condition on line 318 was never true

319 _error(f'--intermediate-package-manifest is mandatory for now') 

320 return TarMember.parse_intermediate_manifest(manifest_path) 

321 

322 

323def main(): 

324 parsed_args = parse_args() 

325 root_dir: str = parsed_args.package_root_dir 

326 output_path: str = parsed_args.package_output_path 

327 

328 mtime: int = parsed_args.source_date_epoch 

329 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ: 

330 sde_raw = os.environ['SOURCE_DATE_EPOCH'] 

331 if sde_raw == '': 

332 # If you replay the example from bash history, but it is the first run in the session, and you did not 

333 # run the "stat" line. 

334 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the" 

335 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line") 

336 mtime = int(sde_raw) 

337 if mtime is None: 

338 mtime = int(time.time()) 

339 

340 data_compression: Compression = COMPRESSIONS[parsed_args.compression_algorithm] 

341 data_compression_cmd = data_compression.as_cmdline(parsed_args) 

342 if parsed_args.uniform_compression: 

343 ctrl_compression = data_compression 

344 ctrl_compression_cmd = data_compression_cmd 

345 else: 

346 ctrl_compression = COMPRESSIONS['gzip'] 

347 ctrl_compression_cmd = COMPRESSIONS['gzip'].as_cmdline(None) 

348 

349 if output_path.endswith('/') or os.path.isdir(output_path): 

350 deb_file = os.path.join(output_path, _compute_output_filename(root_dir, parsed_args.is_udeb)) 

351 else: 

352 deb_file = output_path 

353 

354 pack(deb_file, ctrl_compression, data_compression, root_dir, parsed_args.package_manifest, 

355 mtime, ctrl_compression_cmd, data_compression_cmd) 

356 

357 

358def pack(deb_file: str, ctrl_compression: Compression, data_compression: Compression, root_dir: str, 

359 package_manifest: 'Optional[str]', mtime: int, ctrl_compression_cmd, data_compression_cmd): 

360 

361 data_tar_members = parse_manifest(package_manifest) 

362 members = [ 

363 ArMember('debian-binary', mtime, fixed_binary=b'2.0\n'), 

364 ArMember(ctrl_compression.with_extension('control.tar'), mtime, 

365 write_to_impl=generate_tar_file_member( 

366 _ctrl_tar_members(root_dir, mtime), 

367 ctrl_compression_cmd 

368 )), 

369 ArMember(data_compression.with_extension('data.tar'), mtime, 

370 write_to_impl=generate_tar_file_member( 

371 data_tar_members, 

372 data_compression_cmd 

373 )), 

374 ] 

375 generate_ar_archive(deb_file, mtime, members) 

376 

377 

378if __name__ == '__main__': 

379 main()