Coverage for debputy.py: 0%
179 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-22 14:29 +0100
1#!/usr/bin/python3 -B
2import argparse
3import collections
4import dataclasses
5import json
6import os
7import pathlib
8import subprocess
9import sys
10import textwrap
11import time
12from datetime import datetime
14from debian.debian_support import DpkgArchTable
16from debputy._deb_options_profiles import DebOptionsAndProfiles
17from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable, dpkg_architecture_table
18from debputy.dh_migration import migrate_from_dh, AcceptableMigrationIssues
19from debputy.packages import parse_source_debian_control
20from debputy.version_substitutions import load_source_variables
22# setup PYTHONPATH: add our installation directory.
23sys.path.insert(0, pathlib.Path(__file__).parent)
25from debputy.deb_packaging_support import setup_control_files, build_dbgsym_package_if_relevant
26from debputy.highlevel_manifest import HighLevelManifest, ManifestPathInfoRuleType, ChangeDescription
27from debputy.highlevel_manifest_parser import parse_manifest
28from debputy.intermediate_manifest import TarMember
29from debputy.substitution import Substitution, SubstitutionImpl, NULL_SUBSTITUTION
30from debputy.util import _error
32try:
33 from typing import Union, NoReturn, List, Tuple, Dict, Iterable, Optional, Set, Callable, Any
35 ManifestKey = Union[str, Tuple[str, str]]
36except ImportError:
37 pass
40def parse_args():
41 description = textwrap.dedent(f'''\
42 The `debputy` program is a manifest-based Debian packaging tool.
44 It is used as a part of compiling a source package and transforming it into one or
45 more binary (.deb) packages.
46 ''')
48 parser = argparse.ArgumentParser(
49 description=description,
50 formatter_class=argparse.RawDescriptionHelpFormatter,
51 )
52 parser.add_argument('--package-manifest', dest='package_manifest',
53 action='store', default=None,
54 help='Manifest of file metadata that is not visible in the file-system')
55 parser.add_argument('--source-date-epoch', dest='source_date_epoch',
56 action='store', type=int, default=None,
57 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable'
58 )
59 parser.add_argument('--substitution', dest='substitution',
60 action='store_true', default=None,
61 help='*Some* commands can work without doing path substitution. With this option, substitution'
62 ' is always done. It is a no-op for commands that requires substitution'
63 )
64 parser.add_argument('--no-substitution', dest='substitution',
65 action='store_false', default=None,
66 help='*Some* commands can work without doing path substitution. With this option, substitution'
67 ' is not done. However, this option *cannot* be used with all commands.'
68 )
70 parser.add_argument('-p', '--package', dest='packages',
71 action='append', type=str, default=[],
72 help='The package(s) to act on. Affects default permission normalization rules'
73 )
75 subparsers = parser.add_subparsers(dest='command', required=True)
77 generate_deb_package = subparsers.add_parser('generate-debs-from-binary-staging-directories',
78 help="Generate .deb/.udebs packages from debian/<pkg>")
79 generate_deb_package.add_argument('output', metavar="output",
80 help='Where to place the resulting packages. Should be a directory'
81 )
82 # Added for "help only" - you cannot trigger this option in practice
83 generate_deb_package.add_argument('--', metavar='UPSTREAM_ARGS', action='extend', nargs='+', dest='unused')
85 migrate_from_debhelper = subparsers.add_parser('migrate-from-dh',
86 help='Generate/update manifest from a "dh $@" using package')
87 migrate_from_debhelper.add_argument('--acceptable-migration-issues', dest='acceptable_migration_issues',
88 action='append', type=str, default=[],
89 help='Continue the migration even if this/these issues are detected.'
90 ' Can be set to ALL (in all upper-case) to accept all issues')
92 migrate_from_debhelper.add_argument('--no-act', dest='destructive',
93 action='store_false', default=True,
94 help='Do not perform changes. Existing manifest will not be overridden')
96 show_changes_parser = subparsers.add_parser('show-manifest-changes', help="Show what the manifest will do")
97 show_changes_parser.add_argument('--show-implicit-changes', action='store_true', default=False,
98 dest='show_implicit_changes',
99 help="Show implicit changes (usually directories implied by other actions)")
100 show_changes_parser.add_argument('--show-builtin-rules', action='store_true', default=False,
101 dest='show_builtin_rules',
102 help="Show builtin rules (e.g., default permission normalization)")
104 generator_parser = subparsers.add_parser('debug-generate-intermediate-manifest',
105 help="[Debugging] Generate intermediate manifest for deb_packer")
107 generator_parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR",
108 help='Root directory of the package. Must contain a DEBIAN directory'
109 )
110 generator_parser.add_argument('intermediate_manifest_output', nargs='?', metavar="PATH",
111 help='Path where the intermediate manifest should be placed.'
112 )
114 argv = sys.argv
115 try:
116 i = argv.index('--')
117 upstream_args = argv[i + 1:]
118 argv = argv[:i]
119 except (IndexError, ValueError):
120 upstream_args = []
121 parsed_args = parser.parse_args(argv[1:])
122 setattr(parsed_args, 'upstream_args', upstream_args)
123 setattr(parsed_args, 'packages', frozenset(parsed_args.packages))
125 return parsed_args
128def _serialize_intermediate_manifest(members: Iterable[TarMember]) -> str:
129 serial_format = [m.to_manifest() for m in members]
130 return json.dumps(serial_format)
133def output_intermediate_manifest(manifest_output_file: str, members: Iterable[TarMember]) -> None:
134 with open(manifest_output_file, 'w') as fd:
135 serial_format = [m.to_manifest() for m in members]
136 json.dump(serial_format, fd)
139def _generate_intermediate_manifest(parsed_args, manifest: HighLevelManifest, mtime: int) -> None:
140 output_path = parsed_args.intermediate_manifest_output
141 active_packages = list(manifest.active_packages)
142 if len(active_packages) != 1:
143 _error("This command requires exactly one package (-p/--package")
144 root_dir = parsed_args.package_root_dir
145 package = active_packages[0].name
146 _, intermediate_manifest = manifest.apply_to_binary_staging_directory(package, root_dir, mtime)
147 output_intermediate_manifest(output_path, intermediate_manifest)
150def _print_manifest_change_group(grouped: Dict[ManifestPathInfoRuleType, List[ChangeDescription]],
151 rule_type: ManifestPathInfoRuleType,
152 header: str,
153 footer: str = ''
154 ):
155 changes = grouped.get(rule_type)
156 if changes:
157 print(header)
158 for change_description in changes:
159 print(change_description.pretty_format())
160 print()
161 if footer:
162 print(footer)
163 print()
166def _show_manifest_changes(parsed_args, manifest: HighLevelManifest, mtime: int) -> None:
167 for dctrl_bin in manifest.active_packages:
168 package = dctrl_bin.name
169 package_state = manifest.package_state_for(package)
170 declared_transformations = [t for t in package_state.transformations if not t.is_builtin_rule]
172 if declared_transformations:
173 print("Transformations:")
174 for transformation in declared_transformations:
175 print(f' * {transformation.describe_transformation()}')
176 print()
177 grouped = collections.defaultdict(list)
178 for change_description in sorted(manifest.change_descriptions(package),
179 key=lambda x: x.match_rule.sort_key(),
180 reverse=True,
181 ):
182 path_info = change_description.path_info
183 if path_info.is_builtin_definition and not parsed_args.show_builtin_rules:
184 continue
185 if path_info.is_implicit_definition and not parsed_args.show_implicit_changes:
186 continue
187 grouped[change_description.path_info.rule_type].append(change_description)
189 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_DIRECTORY, 'Directory creation:')
190 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_SYMLINK, 'Symlink creation:')
192 print("Metadata changes (default settings):")
193 print(" * Owner: root (uid: 0)")
194 print(" * Group: root (gid: 0)")
195 print(f" * mtime (clamp/upper bound): {mtime} ({datetime.utcfromtimestamp(mtime)}Z)")
196 print()
197 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.ENSURE_METADATA, 'Metadata changes (path rules):')
199 if package_state.maintscript_snippets:
200 print()
201 print("Maintscript snippets triggered by the manifest:")
202 for script, snippets in package_state.maintscript_snippets.items():
203 print(f" * {script}:")
204 for snippet in snippets:
205 print(f" - triggered by {snippet.definition_source}")
208def _generate_debs_from_binary_staging_directories(parsed_args, manifest: HighLevelManifest, mtime: int) -> None:
209 output_path = parsed_args.output
210 upstream_args = parsed_args.upstream_args
212 for dctrl_bin in manifest.active_packages:
213 package = dctrl_bin.name
214 root_dir = os.path.join('debian', package)
215 fs_root, intermediate_manifest = manifest.apply_to_binary_staging_directory(package, root_dir, mtime)
217 assert dctrl_bin.should_be_acted_on
219 setup_control_files(dctrl_bin, root_dir, fs_root, manifest)
221 build_dbgsym_package_if_relevant(dctrl_bin, upstream_args, output_path)
223 deb_packer = os.path.join(pathlib.Path(__file__).parent, 'deb_packer.py')
224 deb_cmd = [deb_packer, '--intermediate-package-manifest', '-', '--source-date-epoch', str(mtime)]
225 if dctrl_bin.is_udeb:
226 deb_cmd.extend(['--udeb', '-z6', '-Zxz', '-Sextreme'])
227 deb_cmd.extend(upstream_args)
228 deb_cmd.extend(['--build', root_dir, output_path])
230 print(f"Executing for {package}: {deb_cmd}")
231 proc = subprocess.Popen(deb_cmd, stdin=subprocess.PIPE)
232 proc.communicate(_serialize_intermediate_manifest(intermediate_manifest).encode('utf-8'))
233 if proc.returncode != 0:
234 _error(f"{deb_packer} exited with a non-zero exit code!")
237def _migrate_from_dh(parsed_args, manifest: HighLevelManifest, _mtime: int) -> None:
238 acceptable_migration_issues = AcceptableMigrationIssues(
239 frozenset(i for x in parsed_args.acceptable_migration_issues for i in x.split(','))
240 )
241 migrate_from_dh(manifest, acceptable_migration_issues, parsed_args.destructive)
244@dataclasses.dataclass
245class Command:
247 handler: Callable[[Any, HighLevelManifest, int], None]
248 require_substitution: bool = True
250 def create_substitution(self, parsed_args) -> Substitution:
251 requested_subst = parsed_args.substitution
252 if requested_subst is False and self.require_substitution:
253 _error(f"--no-substitution cannot be used with {parsed_args.command}")
254 if self.require_substitution or requested_subst is not False:
255 if not os.path.isfile('debian/changelog'):
256 _error('Expected to be run from a directory containing debian/changelog')
257 v = load_source_variables('debian/changelog')
258 return SubstitutionImpl(extra_substitutions=v)
259 return NULL_SUBSTITUTION
262COMMANDS = {
263 'debug-generate-intermediate-manifest': Command(handler=_generate_intermediate_manifest),
264 'generate-debs-from-binary-staging-directories': Command(handler=_generate_debs_from_binary_staging_directories),
265 'show-manifest-changes': Command(handler=_show_manifest_changes, require_substitution=False),
266 'migrate-from-dh': Command(handler=_migrate_from_dh)
267}
270def main():
271 parsed_args = parse_args()
273 mtime = parsed_args.source_date_epoch
274 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ:
275 sde_raw = os.environ['SOURCE_DATE_EPOCH']
276 if sde_raw == '':
277 # If you replay the example from bash history, but it is the first run in the session, and you did not
278 # run the "stat" line.
279 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the"
280 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line")
281 mtime = int(sde_raw)
282 if mtime is None:
283 mtime = int(time.time())
285 command = COMMANDS[parsed_args.command]
287 packages = parsed_args.packages
288 substitution = command.create_substitution(parsed_args)
289 build_env = DebOptionsAndProfiles.instance()
290 dpkg_architecture_variables = dpkg_architecture_table()
291 dpkg_arch_query_table = DpkgArchTable.load_arch_table()
293 _, binary_packages = parse_source_debian_control(
294 packages, # -p/--package
295 set(), # -N/--no-package
296 False, # -i
297 False, # -a
298 dpkg_architecture_variables=dpkg_architecture_variables,
299 dpkg_arch_query_table=dpkg_arch_query_table,
300 build_env=build_env,
301 )
302 assert packages <= binary_packages.keys()
304 manifest = parse_manifest(parsed_args.package_manifest,
305 binary_packages,
306 substitution,
307 dpkg_architecture_variables,
308 dpkg_arch_query_table,
309 build_env,
310 )
311 command.handler(parsed_args, manifest, mtime)
314if __name__ == '__main__':
315 main()