Coverage for src/gitlabracadabra/packages/destination.py: 85%

1#

2# Copyright (C) 2019-2022 Mathieu Parent <math.parent@gmail.com>

3#

4# This program is free software: you can redistribute it and/or modify

5# it under the terms of the GNU Lesser General Public License as published by

6# the Free Software Foundation, either version 3 of the License, or

7# (at your option) any later version.

8#

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU Lesser General Public License for more details.

13#

14# You should have received a copy of the GNU Lesser General Public License

15# along with this program. If not, see <http://www.gnu.org/licenses/>.

16

17 from __future__ import annotations

18

19 from logging import getLogger

20 from typing import TYPE_CHECKING

21

22 from requests import RequestException, codes

23

24 from gitlabracadabra import __version__ as gitlabracadabra_version

25 from gitlabracadabra.packages.stream import Stream

26 from gitlabracadabra.session import Session

27

28 if TYPE_CHECKING: 28 ↛ 29line 28 didn't jump to line 29 because the condition on line 28 was never true

29 from gitlabracadabra.packages.package_file import PackageFile

30 from gitlabracadabra.packages.source import Source

31

32 logger = getLogger(__name__)

33

34

35 class Destination:

36 """Destination package repository."""

37

38 def __init__(

39 self,

40 *,

41 log_prefix: str = "",

42 ) -> None:

43 """Initialize Destination repository.

44

45 Args:

46 log_prefix: Log prefix.

47 """

48 self._log_prefix = log_prefix

49 self.session = Session()

50 self.session.headers["User-Agent"] = f"gitlabracadabra/{gitlabracadabra_version}"

51

52 def __del__(self) -> None:

53 """Destroy a connection."""

54 self.session.close()

55

56 def import_source(self, source: Source, *, dry_run: bool) -> None:

57 """Import package files from Source.

58

59 Args:

60 source: Source repository.

61 dry_run: Dry run.

62 """

63 try:

64 for package_file in source.package_files:

65 self.try_import_package_file(source, package_file, dry_run=dry_run)

66 except RequestException as err:

67 if err.request: 67 ↛ 77line 67 didn't jump to line 77 because the condition on line 67 was always true

68 logger.warning(

69 "%sError retrieving package files list from %s (%s %s): %s",

70 self._log_prefix,

71 str(source),

72 err.request.method,

73 err.request.url,

74 repr(err),

75 )

76 else:

77 logger.warning(

78 "%sError retrieving package files list from %s: %s",

79 self._log_prefix,

80 str(source),

81 repr(err),

82 )

83

84 def try_import_package_file(self, source: Source, package_file: PackageFile, *, dry_run: bool) -> None:

85 """Try to import one package file, and catch RequestExceptions.

86

87 Args:

88 source: Source repository.

89 package_file: Source package file.

90 dry_run: Dry run.

91 """

92 try:

93 self.import_package_file(source, package_file, dry_run=dry_run)

94 except RequestException as err:

95 if err.request: 95 ↛ 108line 95 didn't jump to line 108 because the condition on line 95 was always true

96 logger.warning(

97 '%sError uploading %s package file "%s" from "%s" version %s (%s %s): %s',

98 self._log_prefix,

99 package_file.package_type,

100 package_file.file_name,

101 package_file.package_name,

102 package_file.package_version,

103 err.request.method,

104 err.request.url,

105 repr(err),

106 )

107 else:

108 logger.warning(

109 '%sError uploading %s package file "%s" from "%s" version %s: %s',

110 self._log_prefix,

111 package_file.package_type,

112 package_file.file_name,

113 package_file.package_name,

114 package_file.package_version,

115 repr(err),

116 )

117

118 def import_package_file(self, source: Source, package_file: PackageFile, *, dry_run: bool) -> None:

119 """Import one package file.

120

121 Args:

122 source: Source repository.

123 package_file: Source package file.

124 dry_run: Dry run.

125 """

126 # Test source exists

127 if not self._source_package_file_exists(source, package_file):

128 return

129

130 # Test destination exists

131 if self._destination_package_file_exists(package_file):

132 return

133

134 # Test dry run

135 if self._dry_run(package_file, dry_run=dry_run):

136 return

137

138 # Upload

139 self._upload_package_file(source, package_file)

140

141 def upload_method(

142 self,

143 package_file: PackageFile, # noqa: ARG002

144 ) -> str:

145 """Get upload HTTP method.

146

147 Args:

148 package_file: Source package file.

149

150 Returns:

151 The upload method.

152 """

153 return "PUT"

154

155 def head_url(self, package_file: PackageFile) -> str:

156 """Get URL to test existence of destination package file with a HEAD request.

157

158 Args:

159 package_file: Source package file.

160

161 Raises:

162 NotImplementedError: This is an abstract method.

163 """

164 raise NotImplementedError

165

166 def upload_url(self, package_file: PackageFile) -> str:

167 """Get URL to upload to.

168

169 Args:

170 package_file: Source package file.

171

172 Returns:

173 The upload URL.

174 """

175 return self.head_url(package_file)

176

177 def files_key(

178 self,

179 package_file: PackageFile, # noqa: ARG002

180 ) -> str | None:

181 """Get files key, to upload to. If None, uploaded as body.

182

183 Args:

184 package_file: Source package file.

185

186 Returns:

187 The files key, or None.

188 """

189 return None

190

191 def _source_package_file_exists(self, source: Source, package_file: PackageFile) -> bool:

192 source_exists_response = source.session.request(

193 "HEAD",

194 package_file.url,

195 )

196 if source_exists_response.status_code == codes["ok"]:

197 return True

198 if source_exists_response.status_code == codes["not_found"]: 198 ↛ 209line 198 didn't jump to line 209 because the condition on line 198 was always true

199 logger.warning(

200 '%sNOT uploading %s package file "%s" from "%s" version %s (%s): source not found',

201 self._log_prefix,

202 package_file.package_type,

203 package_file.file_name,

204 package_file.package_name,

205 package_file.package_version,

206 package_file.url,

207 )

208 return False

209 logger.warning(

210 '%sNOT uploading %s package file "%s" from "%s" version %s (%s): received %i %s with HEAD method on source',

211 self._log_prefix,

212 package_file.package_type,

213 package_file.file_name,

214 package_file.package_name,

215 package_file.package_version,

216 package_file.url,

217 source_exists_response.status_code,

218 source_exists_response.reason,

219 )

220 return False

221

222 def _destination_package_file_exists(self, package_file: PackageFile) -> bool:

223 head_url = self.head_url(package_file)

224 destination_exists_response = self.session.request(

225 "HEAD",

226 head_url,

227 )

228 if destination_exists_response.status_code == codes["ok"]:

229 return True

230 if destination_exists_response.status_code == codes["not_found"]: 230 ↛ 232line 230 didn't jump to line 232 because the condition on line 230 was always true

231 return False

232 logger.warning(

233 '%sUnexpected HTTP status for %s package file "%s" from "%s" version %s (%s): received %i %s with HEAD method on destination',

234 self._log_prefix,

235 package_file.package_type,

236 package_file.file_name,

237 package_file.package_name,

238 package_file.package_version,

239 head_url,

240 destination_exists_response.status_code,

241 destination_exists_response.reason,

242 )

243 return False

244

245 def _dry_run(self, package_file: PackageFile, *, dry_run: bool) -> bool:

246 if dry_run:

247 logger.info(

248 '%sNOT uploading %s package file "%s" from "%s" version %s (%s): Dry run',

249 self._log_prefix,

250 package_file.package_type,

251 package_file.file_name,

252 package_file.package_name,

253 package_file.package_version,

254 package_file.url,

255 )

256 return dry_run

257

258 def _upload_package_file(self, source: Source, package_file: PackageFile) -> None:

259 upload_method = self.upload_method(package_file)

260 upload_url = self.upload_url(package_file)

261 files_key = self.files_key(package_file)

262

263 logger.info(

264 '%sUploading %s package file "%s" from "%s" version %s (%s)',

265 self._log_prefix,

266 package_file.package_type,

267 package_file.file_name,

268 package_file.package_name,

269 package_file.package_version,

270 package_file.url,

271 )

272 download_response = source.session.request(

273 "GET",

274 package_file.url,

275 stream=True,

276 headers={

277 "Accept-Encoding": "*",

278 },

279 )

280

281 if files_key:

282 upload_response = self.session.request(

283 upload_method,

284 upload_url,

285 files={files_key: Stream(download_response)}, # type: ignore

286 )

287 else:

288 upload_response = self.session.request(

289 upload_method,

290 upload_url,

291 data=Stream(download_response),

292 )

293 if upload_response.status_code not in {codes["created"], codes["accepted"]}: 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true

294 logger.warning(

295 '%sError uploading %s package file "%s" from "%s" version %s (%s): %s',

296 self._log_prefix,

297 package_file.package_type,

298 package_file.file_name,

299 package_file.package_name,

300 package_file.package_version,

301 upload_url,

302 upload_response.content,

303 )