Coverage for high-level-manifest-parser.py: 18%

161 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-04 22:52 +0100

1#!/usr/bin/python3 

2import argparse 

3import collections 

4import dataclasses 

5import json 

6import os 

7import pathlib 

8import subprocess 

9import sys 

10import textwrap 

11import time 

12from datetime import datetime 

13 

14from debputy.packages import parse_source_debian_control 

15 

16# setup PYTHONPATH: add our installation directory. 

17sys.path.insert(0, pathlib.Path(__file__).parent) 

18 

19from debputy.deb_packaging_support import setup_control_files, build_dbgsym_package_if_relevant 

20from debputy.highlevel_manifest import HighLevelManifest, ManifestPathInfoRuleType, ChangeDescription 

21from debputy.highlevel_manifest_parser import parse_manifest 

22from debputy.intermediate_manifest import TarMember 

23from debputy.substitution import Substitution, SubstitutionImpl, NULL_SUBSTITUTION 

24from debputy.util import _error 

25 

26try: 

27 from typing import Union, NoReturn, List, Tuple, Dict, Iterable, Optional, Set, Callable, Any 

28 

29 ManifestKey = Union[str, Tuple[str, str]] 

30except ImportError: 

31 pass 

32 

33 

34def parse_args(): 

35 me = os.path.basename(sys.argv[0]) 

36 description = textwrap.dedent(f'''\ 

37 THIS IS A PROTOTYPE tool for testing manifest formats. 

38  

39 Given a binary Debian package root (dpkg-deb --raw-extract some.deb) and a high-level 

40 manifest, it will generate a "intermediate" manifest format that `deb_packer.py` can 

41 consume to built a deb. 

42 

43 Example usage for the prototype: 

44 

45 # SETUP 

46 $ [ "$(whoami)" != 'root' ] || echo "There is no point if you are running this as root" 

47 $ TOOL_DIR=$(pwd) 

48 $ mkdir local-test && cd local-test 

49 $ apt download sudo 

50 $ mv sudo_*.deb original_sudo.deb 

51 $ dpkg-deb --raw-extract original_sudo.deb root 

52 $ chmod 0755 root/usr/bin/sudo 

53 $ echo "Verify that root/usr/bin/sudo is owned by non-root and is 0755" 

54 $ ls -l root/usr/bin/sudo 

55 

56 # USING the prototype 

57 $ SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y) 

58 $ cat > dh-manifest.yaml <<EOF 

59 path-metadata: 

60 usr/bin/sudo: 

61 mode: "04755" 

62 ## owner/group defaults to root. 

63 ## But try to uncomment for added fun 

64 # owner: sys 

65 # group: 3 

66 EOF 

67 # (alternative format - remember to correct the --package-manifest parameter if you want to use this one) 

68 $ cat > install <<EOF 

69 # Try adding owner=sys:sys 

70 usr/bin/sudo mode=04755 

71 EOF 

72 $ SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH "$TOOL_DIR/{me}" -p sudo --package-manifest dh-manifest.yaml generate-intermediate-manifest root intermediate-manifest.json 

73 $ SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH "$TOOL_DIR/deb_packer.py" --intermediate-package-manifest intermediate-manifest.json root . 

74 $ mv sudo_*.deb repacked_sudo.deb 

75 

76 # Verify that usr/bin/sudo is now setuid and owned by root (or whatever owner you specified in the manifest) 

77 $ diffoscope original_sudo.deb repacked_sudo.deb 

78 ''') 

79 

80 parser = argparse.ArgumentParser( 

81 description=description, 

82 formatter_class=argparse.RawDescriptionHelpFormatter, 

83 ) 

84 parser.add_argument('--package-manifest', dest='package_manifest', 

85 action='store', default=None, 

86 help='Manifest of file metadata that is not visible in the file-system') 

87 parser.add_argument('--source-date-epoch', dest='source_date_epoch', 

88 action='store', type=int, default=None, 

89 help='Source date epoch (can also be given via the SOURCE_DATE_EPOCH environ variable' 

90 ) 

91 parser.add_argument('--substitution', dest='substitution', 

92 action='store_true', default=None, 

93 help='*Some* commands can work without doing path substitution. With this option, substitution' 

94 ' is always done. It is a no-op for commands that requires substitution' 

95 ) 

96 parser.add_argument('--no-substitution', dest='substitution', 

97 action='store_false', default=None, 

98 help='*Some* commands can work without doing path substitution. With this option, substitution' 

99 ' is not done. However, this option *cannot* be used with all commands.' 

100 ) 

101 

102 parser.add_argument('-p', '--package', dest='package', 

103 action='store', type=str, default=None, 

104 help='The package this manifest is for. Affects default permission normalization rules' 

105 ) 

106 

107 subparsers = parser.add_subparsers(dest='command', required=True) 

108 

109 generate_deb_package = subparsers.add_parser('generate-deb', 

110 help="Generate a deb package") 

111 generate_deb_package.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR", 

112 help='Root directory of the package. Must contain a DEBIAN directory' 

113 ) 

114 generate_deb_package.add_argument('output', metavar="output", 

115 help='Where to place the resulting deb. Should be a directory' 

116 ) 

117 # Added for "help only" - you cannot trigger this option in practice 

118 generate_deb_package.add_argument('--', metavar='UPSTREAM_ARGS', action='extend', nargs='+', dest='unused') 

119 

120 generator_parser = subparsers.add_parser('generate-intermediate-manifest', 

121 help="Generate intermediate manifest for deb_packer") 

122 

123 generator_parser.add_argument('package_root_dir', metavar="PACKAGE_ROOT_DIR", 

124 help='Root directory of the package. Must contain a DEBIAN directory' 

125 ) 

126 generator_parser.add_argument('intermediate_manifest_output', nargs='?', metavar="PATH", 

127 help='Path where the intermediate manifest should be placed.' 

128 ) 

129 

130 show_changes_parser = subparsers.add_parser('show-manifest-changes', help="Show what the manifest will do") 

131 show_changes_parser.add_argument('--show-implicit-changes', action='store_true', default=False, 

132 dest='show_implicit_changes', 

133 help="Show implicit changes (usually directories implied by other actions)") 

134 show_changes_parser.add_argument('--show-builtin-rules', action='store_true', default=False, 

135 dest='show_builtin_rules', 

136 help="Show builtin rules (e.g., default permission normalization)") 

137 argv = sys.argv 

138 try: 

139 i = argv.index('--') 

140 upstream_args = argv[i + 1:] 

141 argv = argv[:i] 

142 except (IndexError, ValueError): 

143 upstream_args = [] 

144 parsed_args = parser.parse_args(argv[1:]) 

145 setattr(parsed_args, 'upstream_args', upstream_args) 

146 

147 return parsed_args 

148 

149 

150def _serialize_intermediate_manifest(members: Iterable[TarMember]) -> str: 

151 serial_format = [m.to_manifest() for m in members] 

152 return json.dumps(serial_format) 

153 

154 

155def output_intermediate_manifest(manifest_output_file: str, members: Iterable[TarMember]) -> None: 

156 with open(manifest_output_file, 'w') as fd: 

157 serial_format = [m.to_manifest() for m in members] 

158 json.dump(serial_format, fd) 

159 

160 

161def _generate_intermediate_manifest(parsed_args, manifest: HighLevelManifest, mtime: int): 

162 root_dir = parsed_args.package_root_dir 

163 output_path = parsed_args.intermediate_manifest_output 

164 _, intermediate_manifest = manifest.apply_to_directory(root_dir, mtime) 

165 output_intermediate_manifest(output_path, intermediate_manifest) 

166 

167 

168def _print_manifest_change_group(grouped: Dict[ManifestPathInfoRuleType, List[ChangeDescription]], 

169 rule_type: ManifestPathInfoRuleType, 

170 header: str, 

171 footer: str = '' 

172 ): 

173 changes = grouped.get(rule_type) 

174 if changes: 

175 print(header) 

176 for change_description in changes: 

177 print(change_description.pretty_format()) 

178 print() 

179 if footer: 

180 print(footer) 

181 print() 

182 

183 

184def _show_manifest_changes(parsed_args, manifest: HighLevelManifest, mtime: int): 

185 declared_transformations = [t for t in manifest.transformations if not t.is_builtin_rule] 

186 

187 if declared_transformations: 

188 print("Transformations:") 

189 for transformation in declared_transformations: 

190 print(f' * {transformation.describe_transformation()}') 

191 print() 

192 grouped = collections.defaultdict(list) 

193 for change_description in sorted(manifest.change_descriptions(), 

194 key=lambda x: x.match_rule.sort_key(), 

195 reverse=True, 

196 ): 

197 path_info = change_description.path_info 

198 if path_info.is_builtin_definition and not parsed_args.show_builtin_rules: 

199 continue 

200 if path_info.is_implicit_definition and not parsed_args.show_implicit_changes: 

201 continue 

202 grouped[change_description.path_info.rule_type].append(change_description) 

203 

204 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_DIRECTORY, 'Directory creation:') 

205 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.CREATE_SYMLINK, 'Symlink creation:') 

206 

207 print("Metadata changes (default settings):") 

208 print(" * Owner: root (uid: 0)") 

209 print(" * Group: root (gid: 0)") 

210 print(f" * mtime (clamp/upper bound): {mtime} ({datetime.utcfromtimestamp(mtime)}Z)") 

211 print() 

212 _print_manifest_change_group(grouped, ManifestPathInfoRuleType.ENSURE_METADATA, 'Metadata changes (path rules):') 

213 

214 if manifest.snippets: 

215 print() 

216 print("Maintscript snippets triggered by the manifest:") 

217 for script, snippets in manifest.snippets.items(): 

218 print(f" * {script}:") 

219 for snippet in snippets: 

220 print(f" - triggered by {snippet.definition_source}") 

221 

222 

223def _generate_deb(parsed_args, manifest: HighLevelManifest, mtime: int): 

224 root_dir = parsed_args.package_root_dir 

225 output_path = parsed_args.output 

226 upstream_args = parsed_args.upstream_args 

227 package = parsed_args.package 

228 

229 fs_root, intermediate_manifest = manifest.apply_to_directory(root_dir, mtime) 

230 

231 _, binary_packages = parse_source_debian_control( 

232 {package}, # -p/--package 

233 set(), # -N/--no-package 

234 False, # -i 

235 False, # -a 

236 ) 

237 assert package in binary_packages 

238 

239 dctrl_bin = binary_packages[package] 

240 

241 assert dctrl_bin.should_be_acted_on 

242 

243 setup_control_files(dctrl_bin, root_dir, fs_root, manifest) 

244 

245 build_dbgsym_package_if_relevant(dctrl_bin, upstream_args, output_path) 

246 

247 deb_packer = os.path.join(pathlib.Path(__file__).parent, 'deb_packer.py') 

248 deb_cmd = [deb_packer, '--intermediate-package-manifest', '-', '--source-date-epoch', str(mtime)] 

249 if dctrl_bin.is_udeb: 

250 deb_cmd.extend(['--udeb', '-z6', '-Zxz', '-Sextreme']) 

251 deb_cmd.extend(upstream_args) 

252 deb_cmd.extend(['--build', root_dir, output_path]) 

253 

254 print(f"Executing for {package}: {deb_cmd}") 

255 proc = subprocess.Popen(deb_cmd, stdin=subprocess.PIPE) 

256 proc.communicate(_serialize_intermediate_manifest(intermediate_manifest).encode('utf-8')) 

257 if proc.returncode != 0: 

258 _error(f"{deb_packer} exited with a non-zero exit code!") 

259 

260 

261@dataclasses.dataclass 

262class Command: 

263 

264 handler: Callable[[Any, HighLevelManifest, int], None] 

265 require_substitution: bool = True 

266 

267 def create_substitution(self, parsed_args) -> Substitution: 

268 requested_subst = parsed_args.substitution 

269 if requested_subst is False and self.require_substitution: 

270 _error(f"--no-substitution cannot be used with {parsed_args.command}") 

271 if self.require_substitution or requested_subst is not False: 

272 package = parsed_args.package 

273 if package is None: 

274 if self.require_substitution or requested_subst: 

275 _error("The -p/--package parameter is required for this operation") 

276 print('W: -p/--package was not given, using "{PACKAGE}" as placeholder value for "${PACKAGE}"') 

277 package = '{PACKAGE}' 

278 return SubstitutionImpl(extra_substitutions={ 

279 'PACKAGE': package, 

280 }) 

281 return NULL_SUBSTITUTION 

282 

283 

284COMMANDS = { 

285 'generate-intermediate-manifest': Command(handler=_generate_intermediate_manifest), 

286 'generate-deb': Command(handler=_generate_deb), 

287 'show-manifest-changes': Command(handler=_show_manifest_changes, require_substitution=False), 

288} 

289 

290 

291def main(): 

292 parsed_args = parse_args() 

293 

294 mtime = parsed_args.source_date_epoch 

295 if mtime is None and 'SOURCE_DATE_EPOCH' in os.environ: 

296 sde_raw = os.environ['SOURCE_DATE_EPOCH'] 

297 if sde_raw == '': 

298 # If you replay the example from bash history, but it is the first run in the session, and you did not 

299 # run the "stat" line. 

300 _error("SOURCE_DATE_EPOCH is set but empty. If you are using the example, perhaps you omitted the" 

301 " `SOURCE_DATE_EPOCH=$(stat root/DEBIAN/ -c %Y)` line") 

302 mtime = int(sde_raw) 

303 if mtime is None: 

304 mtime = int(time.time()) 

305 

306 command = COMMANDS[parsed_args.command] 

307 substitution = command.create_substitution(parsed_args) 

308 

309 manifest = parse_manifest(parsed_args.package_manifest, substitution) 

310 command.handler(parsed_args, manifest, mtime) 

311 

312 

313if __name__ == '__main__': 

314 main()