Coverage for high-level-manifest-parser.py: 18%
161 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-04 22:52 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-04 22:52 +0100
1#!/usr/bin/python3
2import argparse
3import collections
4import dataclasses
5import json
6import os
7import pathlib
8import subprocess
9import sys
10import textwrap
11import time
12from datetime import datetime
14from debputy.packages import parse_source_debian_control
16# setup PYTHONPATH: add our installation directory.
17sys.path.insert(0, pathlib.Path(__file__).parent)
19from debputy.deb_packaging_support import setup_control_files, build_dbgsym_package_if_relevant
20from debputy.highlevel_manifest import HighLevelManifest, ManifestPathInfoRuleType, ChangeDescription
21from debputy.highlevel_manifest_parser import parse_manifest
22from debputy.intermediate_manifest import TarMember
23from debputy.substitution import Substitution, SubstitutionImpl, NULL_SUBSTITUTION
24from debputy.util import _error
26try:
27 from typing import Union, NoReturn, List, Tuple, Dict, Iterable, Optional, Set, Callable, Any
29 ManifestKey = Union[str, Tuple[str, str]]
30except ImportError:
31 pass
34def parse_args():
35 me = os.path.basename(sys.argv[0])
36 description = textwrap.dedent(f'''\
37 THIS IS A PROTOTYPE tool for testing manifest formats.
39 Given a binary Debian package root (dpkg-deb --raw-extract some.deb) and a high-level
40 manifest, it will generate a "intermediate" manifest format that `deb_packer.py` can
41 consume to built a deb.
43 Example usage for the prototype:
45 # SETUP
46 $ [ "$(whoami)" != 'root' ] || echo "There is no point if you are running this as root"
47 $ TOOL_DIR=$(pwd)
48 $ mkdir local-test && cd local-test
49 $ apt download sudo
50 $ mv sudo_*.deb original_sudo.deb
51 $ dpkg-deb --raw-extract original_sudo.deb root
52 $ chmod 0755 root/usr/bin/sudo
53 $ echo "Verify that root/usr/bin/sudo is owned by non-root and is 0755"
54 $ ls -l root/usr/bin/sudo
56 # USING the prototype
57 $ SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)
58 $ cat > dh-manifest.yaml <<EOF
59 path-metadata:
60 usr/bin/sudo:
61 mode: "04755"
62 ## owner/group defaults to root.
63 ## But try to uncomment for added fun
64 # owner: sys
65 # group: 3
66 EOF
67 # (alternative format - remember to correct the --package-manifest parameter if you want to use this one)
68 $ cat > install <<EOF
69 # Try adding owner=sys:sys
70 usr/bin/sudo mode=04755
71 EOF
72 $ SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH "$TOOL_DIR/{me}" -p sudo --package-manifest dh-manifest.yaml generate-intermediate-manifest root intermediate-manifest.json
73 $ SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH "$TOOL_DIR/deb_packer.py" --intermediate-package-manifest intermediate-manifest.json root .
74 $ mv sudo_*.deb repacked_sudo.deb
76 # Verify that usr/bin/sudo is now setuid and owned by root (or whatever owner you specified in the manifest)
77 $ diffoscope original_sudo.deb repacked_sudo.deb
78 ''')
80 parser = argparse.ArgumentParser(
81 description=description,
82 formatter_class=argparse.RawDescriptionHelpFormatter,
83 )
84 parser.add_argument('--package-manifest', dest='package_manifest',
85 action='store', default=None,
86 help='Manifest of file metadata that is not visible in the file-system')
87 parser.add_argument('--source-date-epoch', dest='source_date_epoch',
88 action='store', type=int, default=None,
89 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable'
90 )
91 parser.add_argument('--substitution', dest='substitution',
92 action='store_true', default=None,
93 help='*Some* commands can work without doing path substitution. With this option, substitution'
94 ' is always done. It is a no-op for commands that requires substitution'
95 )
96 parser.add_argument('--no-substitution', dest='substitution',
97 action='store_false', default=None,
98 help='*Some* commands can work without doing path substitution. With this option, substitution'
99 ' is not done. However, this option *cannot* be used with all commands.'
100 )
102 parser.add_argument('-p', '--package', dest='package',
103 action='store', type=str, default=None,
104 help='The package this manifest is for. Affects default permission normalization rules'
105 )
107 subparsers = parser.add_subparsers(dest='command', required=True)
109 generate_deb_package = subparsers.add_parser('generate-deb',
110 help="Generate a deb package")
111 generate_deb_package.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR",
112 help='Root directory of the package. Must contain a DEBIAN directory'
113 )
114 generate_deb_package.add_argument('output', metavar="output",
115 help='Where to place the resulting deb. Should be a directory'
116 )
117 # Added for "help only" - you cannot trigger this option in practice
118 generate_deb_package.add_argument('--', metavar='UPSTREAM_ARGS', action='extend', nargs='+', dest='unused')
120 generator_parser = subparsers.add_parser('generate-intermediate-manifest',
121 help="Generate intermediate manifest for deb_packer")
123 generator_parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR",
124 help='Root directory of the package. Must contain a DEBIAN directory'
125 )
126 generator_parser.add_argument('intermediate_manifest_output', nargs='?', metavar="PATH",
127 help='Path where the intermediate manifest should be placed.'
128 )
130 show_changes_parser = subparsers.add_parser('show-manifest-changes', help="Show what the manifest will do")
131 show_changes_parser.add_argument('--show-implicit-changes', action='store_true', default=False,
132 dest='show_implicit_changes',
133 help="Show implicit changes (usually directories implied by other actions)")
134 show_changes_parser.add_argument('--show-builtin-rules', action='store_true', default=False,
135 dest='show_builtin_rules',
136 help="Show builtin rules (e.g., default permission normalization)")
137 argv = sys.argv
138 try:
139 i = argv.index('--')
140 upstream_args = argv[i + 1:]
141 argv = argv[:i]
142 except (IndexError, ValueError):
143 upstream_args = []
144 parsed_args = parser.parse_args(argv[1:])
145 setattr(parsed_args, 'upstream_args', upstream_args)
147 return parsed_args
150def _serialize_intermediate_manifest(members: Iterable[TarMember]) -> str:
151 serial_format = [m.to_manifest() for m in members]
152 return json.dumps(serial_format)
155def output_intermediate_manifest(manifest_output_file: str, members: Iterable[TarMember]) -> None:
156 with open(manifest_output_file, 'w') as fd:
157 serial_format = [m.to_manifest() for m in members]
158 json.dump(serial_format, fd)
161def _generate_intermediate_manifest(parsed_args, manifest: HighLevelManifest, mtime: int):
162 root_dir = parsed_args.package_root_dir
163 output_path = parsed_args.intermediate_manifest_output
164 _, intermediate_manifest = manifest.apply_to_directory(root_dir, mtime)
165 output_intermediate_manifest(output_path, intermediate_manifest)
168def _print_manifest_change_group(grouped: Dict[ManifestPathInfoRuleType, List[ChangeDescription]],
169 rule_type: ManifestPathInfoRuleType,
170 header: str,
171 footer: str = ''
172 ):
173 changes = grouped.get(rule_type)
174 if changes:
175 print(header)
176 for change_description in changes:
177 print(change_description.pretty_format())
178 print()
179 if footer:
180 print(footer)
181 print()
184def _show_manifest_changes(parsed_args, manifest: HighLevelManifest, mtime: int):
185 declared_transformations = [t for t in manifest.transformations if not t.is_builtin_rule]
187 if declared_transformations:
188 print("Transformations:")
189 for transformation in declared_transformations:
190 print(f' * {transformation.describe_transformation()}')
191 print()
192 grouped = collections.defaultdict(list)
193 for change_description in sorted(manifest.change_descriptions(),
194 key=lambda x: x.match_rule.sort_key(),
195 reverse=True,
196 ):
197 path_info = change_description.path_info
198 if path_info.is_builtin_definition and not parsed_args.show_builtin_rules:
199 continue
200 if path_info.is_implicit_definition and not parsed_args.show_implicit_changes:
201 continue
202 grouped[change_description.path_info.rule_type].append(change_description)
204 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_DIRECTORY, 'Directory creation:')
205 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_SYMLINK, 'Symlink creation:')
207 print("Metadata changes (default settings):")
208 print(" * Owner: root (uid: 0)")
209 print(" * Group: root (gid: 0)")
210 print(f" * mtime (clamp/upper bound): {mtime} ({datetime.utcfromtimestamp(mtime)}Z)")
211 print()
212 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.ENSURE_METADATA, 'Metadata changes (path rules):')
214 if manifest.snippets:
215 print()
216 print("Maintscript snippets triggered by the manifest:")
217 for script, snippets in manifest.snippets.items():
218 print(f" * {script}:")
219 for snippet in snippets:
220 print(f" - triggered by {snippet.definition_source}")
223def _generate_deb(parsed_args, manifest: HighLevelManifest, mtime: int):
224 root_dir = parsed_args.package_root_dir
225 output_path = parsed_args.output
226 upstream_args = parsed_args.upstream_args
227 package = parsed_args.package
229 fs_root, intermediate_manifest = manifest.apply_to_directory(root_dir, mtime)
231 _, binary_packages = parse_source_debian_control(
232 {package}, # -p/--package
233 set(), # -N/--no-package
234 False, # -i
235 False, # -a
236 )
237 assert package in binary_packages
239 dctrl_bin = binary_packages[package]
241 assert dctrl_bin.should_be_acted_on
243 setup_control_files(dctrl_bin, root_dir, fs_root, manifest)
245 build_dbgsym_package_if_relevant(dctrl_bin, upstream_args, output_path)
247 deb_packer = os.path.join(pathlib.Path(__file__).parent, 'deb_packer.py')
248 deb_cmd = [deb_packer, '--intermediate-package-manifest', '-', '--source-date-epoch', str(mtime)]
249 if dctrl_bin.is_udeb:
250 deb_cmd.extend(['--udeb', '-z6', '-Zxz', '-Sextreme'])
251 deb_cmd.extend(upstream_args)
252 deb_cmd.extend(['--build', root_dir, output_path])
254 print(f"Executing for {package}: {deb_cmd}")
255 proc = subprocess.Popen(deb_cmd, stdin=subprocess.PIPE)
256 proc.communicate(_serialize_intermediate_manifest(intermediate_manifest).encode('utf-8'))
257 if proc.returncode != 0:
258 _error(f"{deb_packer} exited with a non-zero exit code!")
261@dataclasses.dataclass
262class Command:
264 handler: Callable[[Any, HighLevelManifest, int], None]
265 require_substitution: bool = True
267 def create_substitution(self, parsed_args) -> Substitution:
268 requested_subst = parsed_args.substitution
269 if requested_subst is False and self.require_substitution:
270 _error(f"--no-substitution cannot be used with {parsed_args.command}")
271 if self.require_substitution or requested_subst is not False:
272 package = parsed_args.package
273 if package is None:
274 if self.require_substitution or requested_subst:
275 _error("The -p/--package parameter is required for this operation")
276 print('W: -p/--package was not given, using "{PACKAGE}" as placeholder value for "${PACKAGE}"')
277 package = '{PACKAGE}'
278 return SubstitutionImpl(extra_substitutions={
279 'PACKAGE': package,
280 })
281 return NULL_SUBSTITUTION
284COMMANDS = {
285 'generate-intermediate-manifest': Command(handler=_generate_intermediate_manifest),
286 'generate-deb': Command(handler=_generate_deb),
287 'show-manifest-changes': Command(handler=_show_manifest_changes, require_substitution=False),
288}
291def main():
292 parsed_args = parse_args()
294 mtime = parsed_args.source_date_epoch
295 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ:
296 sde_raw = os.environ['SOURCE_DATE_EPOCH']
297 if sde_raw == '':
298 # If you replay the example from bash history, but it is the first run in the session, and you did not
299 # run the "stat" line.
300 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the"
301 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line")
302 mtime = int(sde_raw)
303 if mtime is None:
304 mtime = int(time.time())
306 command = COMMANDS[parsed_args.command]
307 substitution = command.create_substitution(parsed_args)
309 manifest = parse_manifest(parsed_args.package_manifest, substitution)
310 command.handler(parsed_args, manifest, mtime)
313if __name__ == '__main__':
314 main()