Eclipse SUMO - Simulation of Urban MObility
zstr.hpp
Go to the documentation of this file.
1//---------------------------------------------------------
2// Copyright 2015 Ontario Institute for Cancer Research
3// Written by Matei David (matei@cs.toronto.edu)
4//---------------------------------------------------------
5
6// Reference:
7// http://stackoverflow.com/questions/14086417/how-to-write-custom-input-stream-in-c
8
9#pragma once
10
11#include <cassert>
12#include <fstream>
13#include <sstream>
14#include <zlib.h>
15#include <memory>
16#include <iostream>
17#include "strict_fstream.hpp"
18
19namespace zstr
20{
21
22static const std::size_t default_buff_size = static_cast<std::size_t>(1 << 20);
23
26 : public std::ios_base::failure
27{
28public:
29 static std::string error_to_message(z_stream * zstrm_p, int ret)
30 {
31 std::string msg = "zlib: ";
32 switch (ret)
33 {
34 case Z_STREAM_ERROR:
35 msg += "Z_STREAM_ERROR: ";
36 break;
37 case Z_DATA_ERROR:
38 msg += "Z_DATA_ERROR: ";
39 break;
40 case Z_MEM_ERROR:
41 msg += "Z_MEM_ERROR: ";
42 break;
43 case Z_VERSION_ERROR:
44 msg += "Z_VERSION_ERROR: ";
45 break;
46 case Z_BUF_ERROR:
47 msg += "Z_BUF_ERROR: ";
48 break;
49 default:
50 std::ostringstream oss;
51 oss << ret;
52 msg += "[" + oss.str() + "]: ";
53 break;
54 }
55 if (zstrm_p->msg) {
56 msg += zstrm_p->msg;
57 }
58 msg += " ("
59 "next_in: " +
60 std::to_string(uintptr_t(zstrm_p->next_in)) +
61 ", avail_in: " +
62 std::to_string(uintptr_t(zstrm_p->avail_in)) +
63 ", next_out: " +
64 std::to_string(uintptr_t(zstrm_p->next_out)) +
65 ", avail_out: " +
66 std::to_string(uintptr_t(zstrm_p->avail_out)) +
67 ")";
68 return msg;
69 }
70
71 Exception(z_stream * zstrm_p, int ret)
72 : std::ios_base::failure(error_to_message(zstrm_p, ret))
73 {
74 }
75}; // class Exception
76
77namespace detail
78{
79
81 : public z_stream
82{
83public:
84 z_stream_wrapper(bool _is_input, int _level, int _window_bits)
85 : is_input(_is_input)
86 {
87 this->zalloc = nullptr;//Z_NULL
88 this->zfree = nullptr;//Z_NULL
89 this->opaque = nullptr;//Z_NULL
90 int ret;
91 if (is_input)
92 {
93 this->avail_in = 0;
94 this->next_in = nullptr;//Z_NULL
95 ret = inflateInit2(this, _window_bits ? _window_bits : 15+32);
96 }
97 else
98 {
99 ret = deflateInit2(this, _level, Z_DEFLATED, _window_bits ? _window_bits : 15+16, 8, Z_DEFAULT_STRATEGY);
100 }
101 if (ret != Z_OK) throw Exception(this, ret);
102 }
104 {
105 if (is_input)
106 {
107 inflateEnd(this);
108 }
109 else
110 {
111 deflateEnd(this);
112 }
113 }
114private:
116}; // class z_stream_wrapper
117
118} // namespace detail
119
121 : public std::streambuf
122{
123public:
124 istreambuf(std::streambuf * _sbuf_p,
125 std::size_t _buff_size = default_buff_size, bool _auto_detect = true, int _window_bits = 0)
126 : sbuf_p(_sbuf_p),
127 in_buff(),
128 in_buff_start(nullptr),
129 in_buff_end(nullptr),
130 out_buff(),
131 zstrm_p(nullptr),
132 buff_size(_buff_size),
133 auto_detect(_auto_detect),
134 auto_detect_run(false),
135 is_text(false),
136 window_bits(_window_bits)
137 {
138 assert(sbuf_p);
139 in_buff = std::unique_ptr<char[]>(new char[buff_size]);
140 in_buff_start = in_buff.get();
141 in_buff_end = in_buff.get();
142 out_buff = std::unique_ptr<char[]>(new char[buff_size]);
143 setg(out_buff.get(), out_buff.get(), out_buff.get());
144 }
145
146 istreambuf(const istreambuf &) = delete;
147 istreambuf & operator = (const istreambuf &) = delete;
148
149 pos_type seekoff(off_type off, std::ios_base::seekdir dir,
150 std::ios_base::openmode which) override
151 {
152 if (off != 0 || dir != std::ios_base::cur) {
153 return std::streambuf::seekoff(off, dir, which);
154 }
155
156 if (!zstrm_p) {
157 return 0;
158 }
159
160 return static_cast<long int>(zstrm_p->total_out - static_cast<uLong>(in_avail()));
161 }
162
163 std::streambuf::int_type underflow() override
164 {
165 if (this->gptr() == this->egptr())
166 {
167 // pointers for free region in output buffer
168 char * out_buff_free_start = out_buff.get();
169 int tries = 0;
170 do
171 {
172 if (++tries > 1000) {
173 throw std::ios_base::failure("Failed to fill buffer after 1000 tries");
174 }
175
176 // read more input if none available
178 {
179 // empty input buffer: refill from the start
180 in_buff_start = in_buff.get();
181 std::streamsize sz = sbuf_p->sgetn(in_buff.get(), static_cast<std::streamsize>(buff_size));
183 if (in_buff_end == in_buff_start) break; // end of input
184 }
185 // auto detect if the stream contains text or deflate data
187 {
188 auto_detect_run = true;
189 unsigned char b0 = *reinterpret_cast< unsigned char * >(in_buff_start);
190 unsigned char b1 = *reinterpret_cast< unsigned char * >(in_buff_start + 1);
191 // Ref:
192 // http://en.wikipedia.org/wiki/Gzip
193 // http://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like
195 && ((b0 == 0x1F && b1 == 0x8B) // gzip header
196 || (b0 == 0x78 && (b1 == 0x01 // zlib header
197 || b1 == 0x9C
198 || b1 == 0xDA))));
199 }
200 if (is_text)
201 {
202 // simply swap in_buff and out_buff, and adjust pointers
203 assert(in_buff_start == in_buff.get());
205 out_buff_free_start = in_buff_end;
206 in_buff_start = in_buff.get();
207 in_buff_end = in_buff.get();
208 }
209 else
210 {
211 // run inflate() on input
212 if (! zstrm_p) zstrm_p = std::unique_ptr<detail::z_stream_wrapper>(new detail::z_stream_wrapper(true, Z_DEFAULT_COMPRESSION, window_bits));
213 zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(in_buff_start);
214 zstrm_p->avail_in = uint32_t(in_buff_end - in_buff_start);
215 zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff_free_start);
216 zstrm_p->avail_out = uint32_t((out_buff.get() + buff_size) - out_buff_free_start);
217 int ret = inflate(zstrm_p.get(), Z_NO_FLUSH);
218 // process return code
219 if (ret != Z_OK && ret != Z_STREAM_END) throw Exception(zstrm_p.get(), ret);
220 // update in&out pointers following inflate()
221 in_buff_start = reinterpret_cast< decltype(in_buff_start) >(zstrm_p->next_in);
222 in_buff_end = in_buff_start + zstrm_p->avail_in;
223 out_buff_free_start = reinterpret_cast< decltype(out_buff_free_start) >(zstrm_p->next_out);
224 assert(out_buff_free_start + zstrm_p->avail_out == out_buff.get() + buff_size);
225
226 if (ret == Z_STREAM_END) {
227 // if stream ended, deallocate inflator
228 zstrm_p.reset();
229 }
230 }
231 } while (out_buff_free_start == out_buff.get());
232 // 2 exit conditions:
233 // - end of input: there might or might not be output available
234 // - out_buff_free_start != out_buff: output available
235 this->setg(out_buff.get(), out_buff.get(), out_buff_free_start);
236 }
237 return this->gptr() == this->egptr()
238 ? traits_type::eof()
239 : traits_type::to_int_type(*this->gptr());
240 }
241private:
242 std::streambuf * sbuf_p;
243 std::unique_ptr<char[]> in_buff;
246 std::unique_ptr<char[]> out_buff;
247 std::unique_ptr<detail::z_stream_wrapper> zstrm_p;
248 std::size_t buff_size;
253
254}; // class istreambuf
255
257 : public std::streambuf
258{
259public:
260 ostreambuf(std::streambuf * _sbuf_p,
261 std::size_t _buff_size = default_buff_size, int _level = Z_DEFAULT_COMPRESSION, int _window_bits = 0)
262 : sbuf_p(_sbuf_p),
263 in_buff(),
264 out_buff(),
265 zstrm_p(new detail::z_stream_wrapper(false, _level, _window_bits)),
266 buff_size(_buff_size)
267 {
268 assert(sbuf_p);
269 in_buff = std::unique_ptr<char[]>(new char[buff_size]);
270 out_buff = std::unique_ptr<char[]>(new char[buff_size]);
271 setp(in_buff.get(), in_buff.get() + buff_size);
272 }
273
274 ostreambuf(const ostreambuf &) = delete;
275 ostreambuf & operator = (const ostreambuf &) = delete;
276
277 int deflate_loop(int flush)
278 {
279 while (true)
280 {
281 zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff.get());
282 zstrm_p->avail_out = uint32_t(buff_size);
283 int ret = deflate(zstrm_p.get(), flush);
284 if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
285 failed = true;
286 throw Exception(zstrm_p.get(), ret);
287 }
288 std::streamsize sz = sbuf_p->sputn(out_buff.get(), reinterpret_cast< decltype(out_buff.get()) >(zstrm_p->next_out) - out_buff.get());
289 if (sz != reinterpret_cast< decltype(out_buff.get()) >(zstrm_p->next_out) - out_buff.get())
290 {
291 // there was an error in the sink stream
292 return -1;
293 }
294 if (ret == Z_STREAM_END || ret == Z_BUF_ERROR || sz == 0)
295 {
296 break;
297 }
298 }
299 return 0;
300 }
301
302 virtual ~ostreambuf()
303 {
304 // flush the zlib stream
305 //
306 // NOTE: Errors here (sync() return value not 0) are ignored, because we
307 // cannot throw in a destructor. This mirrors the behaviour of
308 // std::basic_filebuf::~basic_filebuf(). To see an exception on error,
309 // close the ofstream with an explicit call to close(), and do not rely
310 // on the implicit call in the destructor.
311 //
312 if (!failed) try {
313 sync();
314 } catch (...) {}
315 }
316 std::streambuf::int_type overflow(std::streambuf::int_type c = traits_type::eof()) override
317 {
318 zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(pbase());
319 zstrm_p->avail_in = uint32_t(pptr() - pbase());
320 while (zstrm_p->avail_in > 0)
321 {
322 int r = deflate_loop(Z_NO_FLUSH);
323 if (r != 0)
324 {
325 setp(nullptr, nullptr);
326 return traits_type::eof();
327 }
328 }
329 setp(in_buff.get(), in_buff.get() + buff_size);
330 return traits_type::eq_int_type(c, traits_type::eof()) ? traits_type::eof() : sputc(char_type(c));
331 }
332 int sync() override
333 {
334 // first, call overflow to clear in_buff
335 overflow();
336 if (! pptr()) return -1;
337 // then, call deflate asking to finish the zlib stream
338 zstrm_p->next_in = nullptr;
339 zstrm_p->avail_in = 0;
340 if (deflate_loop(Z_FINISH) != 0) return -1;
341 deflateReset(zstrm_p.get());
342 return 0;
343 }
344private:
345 std::streambuf * sbuf_p = nullptr;
346 std::unique_ptr<char[]> in_buff;
347 std::unique_ptr<char[]> out_buff;
348 std::unique_ptr<detail::z_stream_wrapper> zstrm_p;
349 std::size_t buff_size;
350 bool failed = false;
351
352}; // class ostreambuf
353
355 : public std::istream
356{
357public:
358 istream(std::istream & is,
359 std::size_t _buff_size = default_buff_size, bool _auto_detect = true, int _window_bits = 0)
360 : std::istream(new istreambuf(is.rdbuf(), _buff_size, _auto_detect, _window_bits))
361 {
362 exceptions(std::ios_base::badbit);
363 }
364 explicit istream(std::streambuf * sbuf_p)
365 : std::istream(new istreambuf(sbuf_p))
366 {
367 exceptions(std::ios_base::badbit);
368 }
369 virtual ~istream()
370 {
371 delete rdbuf();
372 }
373}; // class istream
374
376 : public std::ostream
377{
378public:
379 ostream(std::ostream & os,
380 std::size_t _buff_size = default_buff_size, int _level = Z_DEFAULT_COMPRESSION, int _window_bits = 0)
381 : std::ostream(new ostreambuf(os.rdbuf(), _buff_size, _level, _window_bits))
382 {
383 exceptions(std::ios_base::badbit);
384 }
385 explicit ostream(std::streambuf * sbuf_p)
386 : std::ostream(new ostreambuf(sbuf_p))
387 {
388 exceptions(std::ios_base::badbit);
389 }
390 virtual ~ostream()
391 {
392 delete rdbuf();
393 }
394}; // class ostream
395
396namespace detail
397{
398
399template < typename FStream_Type >
401{
402 strict_fstream_holder(const std::string& filename, std::ios_base::openmode mode = std::ios_base::in)
403 : _fs(filename, mode)
404 {}
406 FStream_Type _fs {};
407}; // class strict_fstream_holder
408
409} // namespace detail
410
412 : private detail::strict_fstream_holder< strict_fstream::ifstream >,
413 public std::istream
414{
415public:
416 explicit ifstream(const std::string filename, std::ios_base::openmode mode = std::ios_base::in, size_t buff_size = default_buff_size)
417 : detail::strict_fstream_holder< strict_fstream::ifstream >(filename, mode),
418 std::istream(new istreambuf(_fs.rdbuf(), buff_size))
419 {
420 exceptions(std::ios_base::badbit);
421 }
422 explicit ifstream(): detail::strict_fstream_holder< strict_fstream::ifstream >(), std::istream(new istreambuf(_fs.rdbuf())){}
423 void close() {
424 _fs.close();
425 }
426 // void open(const std::string filename, std::ios_base::openmode mode = std::ios_base::in) {
427 // _fs.open(filename, mode);
428 // std::istream::operator=(std::istream(new istreambuf(_fs.rdbuf())));
429 // }
430 bool is_open() const {
431 return _fs.is_open();
432 }
433 virtual ~ifstream()
434 {
435 if (_fs.is_open()) close();
436 if (rdbuf()) delete rdbuf();
437 }
438
440 std::streampos compressed_tellg()
441 {
442 return _fs.tellg();
443 }
444}; // class ifstream
445
447 : private detail::strict_fstream_holder< strict_fstream::ofstream >,
448 public std::ostream
449{
450public:
451 explicit ofstream(const std::string filename, std::ios_base::openmode mode = std::ios_base::out,
452 int level = Z_DEFAULT_COMPRESSION, size_t buff_size = default_buff_size)
453 : detail::strict_fstream_holder< strict_fstream::ofstream >(filename, mode | std::ios_base::binary),
454 std::ostream(new ostreambuf(_fs.rdbuf(), buff_size, level))
455 {
456 exceptions(std::ios_base::badbit);
457 }
458 explicit ofstream(): detail::strict_fstream_holder< strict_fstream::ofstream >(), std::ostream(new ostreambuf(_fs.rdbuf())){}
459 void close() {
460 std::ostream::flush();
461 _fs.close();
462 }
463 // void open(const std::string filename, std::ios_base::openmode mode = std::ios_base::out, int level = Z_DEFAULT_COMPRESSION) {
464 // flush();
465 // _fs.open(filename, mode | std::ios_base::binary);
466 // std::ostream::operator=(std::ostream(new ostreambuf(_fs.rdbuf(), default_buff_size, level)));
467 // }
468 bool is_open() const {
469 return _fs.is_open();
470 }
472 std::ostream::flush();
473 _fs.flush();
474 return *this;
475 }
476 virtual ~ofstream()
477 {
478 if (_fs.is_open()) close();
479 if (rdbuf()) delete rdbuf();
480 }
481
482 // Return the position within the compressed file (wrapped filestream)
483 std::streampos compressed_tellp()
484 {
485 return _fs.tellp();
486 }
487}; // class ofstream
488
489} // namespace zstr
490
Exception class thrown by failed zlib operations.
Definition: zstr.hpp:27
static std::string error_to_message(z_stream *zstrm_p, int ret)
Definition: zstr.hpp:29
Exception(z_stream *zstrm_p, int ret)
Definition: zstr.hpp:71
z_stream_wrapper(bool _is_input, int _level, int _window_bits)
Definition: zstr.hpp:84
std::streampos compressed_tellg()
Return the position within the compressed file (wrapped filestream)
Definition: zstr.hpp:440
void close()
Definition: zstr.hpp:423
ifstream(const std::string filename, std::ios_base::openmode mode=std::ios_base::in, size_t buff_size=default_buff_size)
Definition: zstr.hpp:416
bool is_open() const
Definition: zstr.hpp:430
virtual ~ifstream()
Definition: zstr.hpp:433
virtual ~istream()
Definition: zstr.hpp:369
istream(std::streambuf *sbuf_p)
Definition: zstr.hpp:364
istream(std::istream &is, std::size_t _buff_size=default_buff_size, bool _auto_detect=true, int _window_bits=0)
Definition: zstr.hpp:358
bool auto_detect
Definition: zstr.hpp:249
istreambuf(std::streambuf *_sbuf_p, std::size_t _buff_size=default_buff_size, bool _auto_detect=true, int _window_bits=0)
Definition: zstr.hpp:124
char * in_buff_end
Definition: zstr.hpp:245
std::unique_ptr< detail::z_stream_wrapper > zstrm_p
Definition: zstr.hpp:247
std::unique_ptr< char[]> out_buff
Definition: zstr.hpp:246
istreambuf(const istreambuf &)=delete
std::unique_ptr< char[]> in_buff
Definition: zstr.hpp:243
char * in_buff_start
Definition: zstr.hpp:244
std::size_t buff_size
Definition: zstr.hpp:248
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) override
Definition: zstr.hpp:149
bool auto_detect_run
Definition: zstr.hpp:250
std::streambuf::int_type underflow() override
Definition: zstr.hpp:163
std::streambuf * sbuf_p
Definition: zstr.hpp:242
istreambuf & operator=(const istreambuf &)=delete
std::streampos compressed_tellp()
Definition: zstr.hpp:483
ofstream & flush()
Definition: zstr.hpp:471
ofstream(const std::string filename, std::ios_base::openmode mode=std::ios_base::out, int level=Z_DEFAULT_COMPRESSION, size_t buff_size=default_buff_size)
Definition: zstr.hpp:451
void close()
Definition: zstr.hpp:459
bool is_open() const
Definition: zstr.hpp:468
virtual ~ofstream()
Definition: zstr.hpp:476
ostream(std::ostream &os, std::size_t _buff_size=default_buff_size, int _level=Z_DEFAULT_COMPRESSION, int _window_bits=0)
Definition: zstr.hpp:379
ostream(std::streambuf *sbuf_p)
Definition: zstr.hpp:385
virtual ~ostream()
Definition: zstr.hpp:390
ostreambuf & operator=(const ostreambuf &)=delete
int deflate_loop(int flush)
Definition: zstr.hpp:277
std::streambuf * sbuf_p
Definition: zstr.hpp:345
std::size_t buff_size
Definition: zstr.hpp:349
std::unique_ptr< char[]> in_buff
Definition: zstr.hpp:346
std::unique_ptr< char[]> out_buff
Definition: zstr.hpp:347
ostreambuf(const ostreambuf &)=delete
ostreambuf(std::streambuf *_sbuf_p, std::size_t _buff_size=default_buff_size, int _level=Z_DEFAULT_COMPRESSION, int _window_bits=0)
Definition: zstr.hpp:260
std::streambuf::int_type overflow(std::streambuf::int_type c=traits_type::eof()) override
Definition: zstr.hpp:316
int sync() override
Definition: zstr.hpp:332
virtual ~ostreambuf()
Definition: zstr.hpp:302
std::unique_ptr< detail::z_stream_wrapper > zstrm_p
Definition: zstr.hpp:348
auto get(const nlohmann::detail::iteration_proxy_value< IteratorType > &i) -> decltype(i.key())
Definition: json.hpp:4451
NLOHMANN_BASIC_JSON_TPL_DECLARATION std::string to_string(const NLOHMANN_BASIC_JSON_TPL &j)
user-defined to_string function for JSON values
Definition: json.hpp:21838
Definition: json.hpp:4471
NLOHMANN_BASIC_JSON_TPL_DECLARATION void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL &j1, nlohmann::NLOHMANN_BASIC_JSON_TPL &j2) noexcept(//NOLINT(readability-inconsistent-declaration-parameter-name) is_nothrow_move_constructible< nlohmann::NLOHMANN_BASIC_JSON_TPL >::value &&//NOLINT(misc-redundant-expression) is_nothrow_move_assignable< nlohmann::NLOHMANN_BASIC_JSON_TPL >::value)
exchanges the values of two JSON objects
Definition: json.hpp:21884
Definition: zstr.hpp:20
static const std::size_t default_buff_size
Definition: zstr.hpp:22
strict_fstream_holder(const std::string &filename, std::ios_base::openmode mode=std::ios_base::in)
Definition: zstr.hpp:402