Coverage for debputy.py: 0%

179 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-22 14:29 +0100

1#!/usr/bin/python3 -B 

2import argparse 

3import collections 

4import dataclasses 

5import json 

6import os 

7import pathlib 

8import subprocess 

9import sys 

10import textwrap 

11import time 

12from datetime import datetime 

13 

14from debian.debian_support import DpkgArchTable 

15 

16from debputy._deb_options_profiles import DebOptionsAndProfiles 

17from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable, dpkg_architecture_table 

18from debputy.dh_migration import migrate_from_dh, AcceptableMigrationIssues 

19from debputy.packages import parse_source_debian_control 

20from debputy.version_substitutions import load_source_variables 

21 

22# setup PYTHONPATH: add our installation directory. 

23sys.path.insert(0, pathlib.Path(__file__).parent) 

24 

25from debputy.deb_packaging_support import setup_control_files, build_dbgsym_package_if_relevant 

26from debputy.highlevel_manifest import HighLevelManifest, ManifestPathInfoRuleType, ChangeDescription 

27from debputy.highlevel_manifest_parser import parse_manifest 

28from debputy.intermediate_manifest import TarMember 

29from debputy.substitution import Substitution, SubstitutionImpl, NULL_SUBSTITUTION 

30from debputy.util import _error 

31 

32try: 

33 from typing import Union, NoReturn, List, Tuple, Dict, Iterable, Optional, Set, Callable, Any 

34 

35 ManifestKey = Union[str, Tuple[str, str]] 

36except ImportError: 

37 pass 

38 

39 

40def parse_args(): 

41 description = textwrap.dedent(f'''\ 

42 The `debputy` program is a manifest-based Debian packaging tool.  

43  

44 It is used as a part of compiling a source package and transforming it into one or 

45 more binary (.deb) packages. 

46 ''') 

47 

48 parser = argparse.ArgumentParser( 

49 description=description, 

50 formatter_class=argparse.RawDescriptionHelpFormatter, 

51 ) 

52 parser.add_argument('--package-manifest', dest='package_manifest', 

53 action='store', default=None, 

54 help='Manifest of file metadata that is not visible in the file-system') 

55 parser.add_argument('--source-date-epoch', dest='source_date_epoch', 

56 action='store', type=int, default=None, 

57 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable' 

58 ) 

59 parser.add_argument('--substitution', dest='substitution', 

60 action='store_true', default=None, 

61 help='*Some* commands can work without doing path substitution. With this option, substitution' 

62 ' is always done. It is a no-op for commands that requires substitution' 

63 ) 

64 parser.add_argument('--no-substitution', dest='substitution', 

65 action='store_false', default=None, 

66 help='*Some* commands can work without doing path substitution. With this option, substitution' 

67 ' is not done. However, this option *cannot* be used with all commands.' 

68 ) 

69 

70 parser.add_argument('-p', '--package', dest='packages', 

71 action='append', type=str, default=[], 

72 help='The package(s) to act on. Affects default permission normalization rules' 

73 ) 

74 

75 subparsers = parser.add_subparsers(dest='command', required=True) 

76 

77 generate_deb_package = subparsers.add_parser('generate-debs-from-binary-staging-directories', 

78 help="Generate .deb/.udebs packages from debian/<pkg>") 

79 generate_deb_package.add_argument('output', metavar="output", 

80 help='Where to place the resulting packages. Should be a directory' 

81 ) 

82 # Added for "help only" - you cannot trigger this option in practice 

83 generate_deb_package.add_argument('--', metavar='UPSTREAM_ARGS', action='extend', nargs='+', dest='unused') 

84 

85 migrate_from_debhelper = subparsers.add_parser('migrate-from-dh', 

86 help='Generate/update manifest from a "dh $@" using package') 

87 migrate_from_debhelper.add_argument('--acceptable-migration-issues', dest='acceptable_migration_issues', 

88 action='append', type=str, default=[], 

89 help='Continue the migration even if this/these issues are detected.' 

90 ' Can be set to ALL (in all upper-case) to accept all issues') 

91 

92 migrate_from_debhelper.add_argument('--no-act', dest='destructive', 

93 action='store_false', default=True, 

94 help='Do not perform changes. Existing manifest will not be overridden') 

95 

96 show_changes_parser = subparsers.add_parser('show-manifest-changes', help="Show what the manifest will do") 

97 show_changes_parser.add_argument('--show-implicit-changes', action='store_true', default=False, 

98 dest='show_implicit_changes', 

99 help="Show implicit changes (usually directories implied by other actions)") 

100 show_changes_parser.add_argument('--show-builtin-rules', action='store_true', default=False, 

101 dest='show_builtin_rules', 

102 help="Show builtin rules (e.g., default permission normalization)") 

103 

104 generator_parser = subparsers.add_parser('debug-generate-intermediate-manifest', 

105 help="[Debugging] Generate intermediate manifest for deb_packer") 

106 

107 generator_parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR", 

108 help='Root directory of the package. Must contain a DEBIAN directory' 

109 ) 

110 generator_parser.add_argument('intermediate_manifest_output', nargs='?', metavar="PATH", 

111 help='Path where the intermediate manifest should be placed.' 

112 ) 

113 

114 argv = sys.argv 

115 try: 

116 i = argv.index('--') 

117 upstream_args = argv[i + 1:] 

118 argv = argv[:i] 

119 except (IndexError, ValueError): 

120 upstream_args = [] 

121 parsed_args = parser.parse_args(argv[1:]) 

122 setattr(parsed_args, 'upstream_args', upstream_args) 

123 setattr(parsed_args, 'packages', frozenset(parsed_args.packages)) 

124 

125 return parsed_args 

126 

127 

128def _serialize_intermediate_manifest(members: Iterable[TarMember]) -> str: 

129 serial_format = [m.to_manifest() for m in members] 

130 return json.dumps(serial_format) 

131 

132 

133def output_intermediate_manifest(manifest_output_file: str, members: Iterable[TarMember]) -> None: 

134 with open(manifest_output_file, 'w') as fd: 

135 serial_format = [m.to_manifest() for m in members] 

136 json.dump(serial_format, fd) 

137 

138 

139def _generate_intermediate_manifest(parsed_args, manifest: HighLevelManifest, mtime: int) -> None: 

140 output_path = parsed_args.intermediate_manifest_output 

141 active_packages = list(manifest.active_packages) 

142 if len(active_packages) != 1: 

143 _error("This command requires exactly one package (-p/--package") 

144 root_dir = parsed_args.package_root_dir 

145 package = active_packages[0].name 

146 _, intermediate_manifest = manifest.apply_to_binary_staging_directory(package, root_dir, mtime) 

147 output_intermediate_manifest(output_path, intermediate_manifest) 

148 

149 

150def _print_manifest_change_group(grouped: Dict[ManifestPathInfoRuleType, List[ChangeDescription]], 

151 rule_type: ManifestPathInfoRuleType, 

152 header: str, 

153 footer: str = '' 

154 ): 

155 changes = grouped.get(rule_type) 

156 if changes: 

157 print(header) 

158 for change_description in changes: 

159 print(change_description.pretty_format()) 

160 print() 

161 if footer: 

162 print(footer) 

163 print() 

164 

165 

166def _show_manifest_changes(parsed_args, manifest: HighLevelManifest, mtime: int) -> None: 

167 for dctrl_bin in manifest.active_packages: 

168 package = dctrl_bin.name 

169 package_state = manifest.package_state_for(package) 

170 declared_transformations = [t for t in package_state.transformations if not t.is_builtin_rule] 

171 

172 if declared_transformations: 

173 print("Transformations:") 

174 for transformation in declared_transformations: 

175 print(f' * {transformation.describe_transformation()}') 

176 print() 

177 grouped = collections.defaultdict(list) 

178 for change_description in sorted(manifest.change_descriptions(package), 

179 key=lambda x: x.match_rule.sort_key(), 

180 reverse=True, 

181 ): 

182 path_info = change_description.path_info 

183 if path_info.is_builtin_definition and not parsed_args.show_builtin_rules: 

184 continue 

185 if path_info.is_implicit_definition and not parsed_args.show_implicit_changes: 

186 continue 

187 grouped[change_description.path_info.rule_type].append(change_description) 

188 

189 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_DIRECTORY, 'Directory creation:') 

190 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_SYMLINK, 'Symlink creation:') 

191 

192 print("Metadata changes (default settings):") 

193 print(" * Owner: root (uid: 0)") 

194 print(" * Group: root (gid: 0)") 

195 print(f" * mtime (clamp/upper bound): {mtime} ({datetime.utcfromtimestamp(mtime)}Z)") 

196 print() 

197 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.ENSURE_METADATA, 'Metadata changes (path rules):') 

198 

199 if package_state.maintscript_snippets: 

200 print() 

201 print("Maintscript snippets triggered by the manifest:") 

202 for script, snippets in package_state.maintscript_snippets.items(): 

203 print(f" * {script}:") 

204 for snippet in snippets: 

205 print(f" - triggered by {snippet.definition_source}") 

206 

207 

208def _generate_debs_from_binary_staging_directories(parsed_args, manifest: HighLevelManifest, mtime: int) -> None: 

209 output_path = parsed_args.output 

210 upstream_args = parsed_args.upstream_args 

211 

212 for dctrl_bin in manifest.active_packages: 

213 package = dctrl_bin.name 

214 root_dir = os.path.join('debian', package) 

215 fs_root, intermediate_manifest = manifest.apply_to_binary_staging_directory(package, root_dir, mtime) 

216 

217 assert dctrl_bin.should_be_acted_on 

218 

219 setup_control_files(dctrl_bin, root_dir, fs_root, manifest) 

220 

221 build_dbgsym_package_if_relevant(dctrl_bin, upstream_args, output_path) 

222 

223 deb_packer = os.path.join(pathlib.Path(__file__).parent, 'deb_packer.py') 

224 deb_cmd = [deb_packer, '--intermediate-package-manifest', '-', '--source-date-epoch', str(mtime)] 

225 if dctrl_bin.is_udeb: 

226 deb_cmd.extend(['--udeb', '-z6', '-Zxz', '-Sextreme']) 

227 deb_cmd.extend(upstream_args) 

228 deb_cmd.extend(['--build', root_dir, output_path]) 

229 

230 print(f"Executing for {package}: {deb_cmd}") 

231 proc = subprocess.Popen(deb_cmd, stdin=subprocess.PIPE) 

232 proc.communicate(_serialize_intermediate_manifest(intermediate_manifest).encode('utf-8')) 

233 if proc.returncode != 0: 

234 _error(f"{deb_packer} exited with a non-zero exit code!") 

235 

236 

237def _migrate_from_dh(parsed_args, manifest: HighLevelManifest, _mtime: int) -> None: 

238 acceptable_migration_issues = AcceptableMigrationIssues( 

239 frozenset(i for x in parsed_args.acceptable_migration_issues for i in x.split(',')) 

240 ) 

241 migrate_from_dh(manifest, acceptable_migration_issues, parsed_args.destructive) 

242 

243 

244@dataclasses.dataclass 

245class Command: 

246 

247 handler: Callable[[Any, HighLevelManifest, int], None] 

248 require_substitution: bool = True 

249 

250 def create_substitution(self, parsed_args) -> Substitution: 

251 requested_subst = parsed_args.substitution 

252 if requested_subst is False and self.require_substitution: 

253 _error(f"--no-substitution cannot be used with {parsed_args.command}") 

254 if self.require_substitution or requested_subst is not False: 

255 if not os.path.isfile('debian/changelog'): 

256 _error('Expected to be run from a directory containing debian/changelog') 

257 v = load_source_variables('debian/changelog') 

258 return SubstitutionImpl(extra_substitutions=v) 

259 return NULL_SUBSTITUTION 

260 

261 

262COMMANDS = { 

263 'debug-generate-intermediate-manifest': Command(handler=_generate_intermediate_manifest), 

264 'generate-debs-from-binary-staging-directories': Command(handler=_generate_debs_from_binary_staging_directories), 

265 'show-manifest-changes': Command(handler=_show_manifest_changes, require_substitution=False), 

266 'migrate-from-dh': Command(handler=_migrate_from_dh) 

267} 

268 

269 

270def main(): 

271 parsed_args = parse_args() 

272 

273 mtime = parsed_args.source_date_epoch 

274 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ: 

275 sde_raw = os.environ['SOURCE_DATE_EPOCH'] 

276 if sde_raw == '': 

277 # If you replay the example from bash history, but it is the first run in the session, and you did not 

278 # run the "stat" line. 

279 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the" 

280 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line") 

281 mtime = int(sde_raw) 

282 if mtime is None: 

283 mtime = int(time.time()) 

284 

285 command = COMMANDS[parsed_args.command] 

286 

287 packages = parsed_args.packages 

288 substitution = command.create_substitution(parsed_args) 

289 build_env = DebOptionsAndProfiles.instance() 

290 dpkg_architecture_variables = dpkg_architecture_table() 

291 dpkg_arch_query_table = DpkgArchTable.load_arch_table() 

292 

293 _, binary_packages = parse_source_debian_control( 

294 packages, # -p/--package 

295 set(), # -N/--no-package 

296 False, # -i 

297 False, # -a 

298 dpkg_architecture_variables=dpkg_architecture_variables, 

299 dpkg_arch_query_table=dpkg_arch_query_table, 

300 build_env=build_env, 

301 ) 

302 assert packages <= binary_packages.keys() 

303 

304 manifest = parse_manifest(parsed_args.package_manifest, 

305 binary_packages, 

306 substitution, 

307 dpkg_architecture_variables, 

308 dpkg_arch_query_table, 

309 build_env, 

310 ) 

311 command.handler(parsed_args, manifest, mtime) 

312 

313 

314if __name__ == '__main__': 

315 main()