Libosmium  2.5.4
Fast and flexible C++ library for working with OpenStreetMap data
gzip_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_GZIP_COMPRESSION_HPP
2 #define OSMIUM_IO_GZIP_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
45 #include <stdexcept>
46 #include <string>
47 
48 #include <errno.h>
49 #include <zlib.h>
50 
52 #include <osmium/io/error.hpp>
55 #include <osmium/util/cast.hpp>
57 
58 namespace osmium {
59 
64  struct gzip_error : public io_error {
65 
68 
69  gzip_error(const std::string& what, int error_code) :
70  io_error(what),
71  gzip_error_code(error_code),
72  system_errno(error_code == Z_ERRNO ? errno : 0) {
73  }
74 
75  }; // struct gzip_error
76 
77  namespace io {
78 
79  namespace detail {
80 
81  OSMIUM_NORETURN inline void throw_gzip_error(gzFile gzfile, const char* msg, int zlib_error = 0) {
82  std::string error("gzip error: ");
83  error += msg;
84  error += ": ";
85  int errnum = zlib_error;
86  if (zlib_error) {
87  error += std::to_string(zlib_error);
88  } else {
89  error += ::gzerror(gzfile, &errnum);
90  }
91  throw osmium::gzip_error(error, errnum);
92  }
93 
94  } // namespace detail
95 
96  class GzipCompressor : public Compressor {
97 
98  int m_fd;
99  gzFile m_gzfile;
100 
101  public:
102 
103  explicit GzipCompressor(int fd, fsync sync) :
104  Compressor(sync),
105  m_fd(dup(fd)),
106  m_gzfile(::gzdopen(fd, "w")) {
107  if (!m_gzfile) {
108  detail::throw_gzip_error(m_gzfile, "write initialization failed");
109  }
110  }
111 
112  ~GzipCompressor() noexcept final {
113  try {
114  close();
115  } catch (...) {
116  // Ignore any exceptions because destructor must not throw.
117  }
118  }
119 
120  void write(const std::string& data) final {
121  if (!data.empty()) {
122  int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast_with_assert<unsigned int>(data.size()));
123  if (nwrite == 0) {
124  detail::throw_gzip_error(m_gzfile, "write failed");
125  }
126  }
127  }
128 
129  void close() final {
130  if (m_gzfile) {
131  int result = ::gzclose(m_gzfile);
132  m_gzfile = nullptr;
133  if (result != Z_OK) {
134  detail::throw_gzip_error(m_gzfile, "write close failed", result);
135  }
136  if (do_fsync()) {
137  osmium::io::detail::reliable_fsync(m_fd);
138  }
139  osmium::io::detail::reliable_close(m_fd);
140  }
141  }
142 
143  }; // class GzipCompressor
144 
146 
147  gzFile m_gzfile;
148 
149  public:
150 
151  explicit GzipDecompressor(int fd) :
152  Decompressor(),
153  m_gzfile(::gzdopen(fd, "r")) {
154  if (!m_gzfile) {
155  detail::throw_gzip_error(m_gzfile, "read initialization failed");
156  }
157  }
158 
159  ~GzipDecompressor() noexcept final {
160  try {
161  close();
162  } catch (...) {
163  // Ignore any exceptions because destructor must not throw.
164  }
165  }
166 
167  std::string read() final {
168  std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0');
169  int nread = ::gzread(m_gzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<unsigned int>(buffer.size()));
170  if (nread < 0) {
171  detail::throw_gzip_error(m_gzfile, "read failed");
172  }
173  buffer.resize(static_cast<std::string::size_type>(nread));
174  return buffer;
175  }
176 
177  void close() final {
178  if (m_gzfile) {
179  int result = ::gzclose(m_gzfile);
180  m_gzfile = nullptr;
181  if (result != Z_OK) {
182  detail::throw_gzip_error(m_gzfile, "read close failed", result);
183  }
184  }
185  }
186 
187  }; // class GzipDecompressor
188 
190 
191  const char* m_buffer;
193  z_stream m_zstream;
194 
195  public:
196 
197  GzipBufferDecompressor(const char* buffer, size_t size) :
198  m_buffer(buffer),
199  m_buffer_size(size),
200  m_zstream() {
201  m_zstream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(buffer));
202  m_zstream.avail_in = static_cast_with_assert<unsigned int>(size);
203  int result = inflateInit2(&m_zstream, MAX_WBITS | 32);
204  if (result != Z_OK) {
205  std::string message("gzip error: decompression init failed: ");
206  if (m_zstream.msg) {
207  message.append(m_zstream.msg);
208  }
209  throw osmium::gzip_error(message, result);
210  }
211  }
212 
213  ~GzipBufferDecompressor() noexcept final {
214  try {
215  close();
216  } catch (...) {
217  // Ignore any exceptions because destructor must not throw.
218  }
219  }
220 
221  std::string read() final {
222  std::string output;
223 
224  if (m_buffer) {
225  const size_t buffer_size = 10240;
226  output.append(buffer_size, '\0');
227  m_zstream.next_out = reinterpret_cast<unsigned char*>(const_cast<char*>(output.data()));
228  m_zstream.avail_out = buffer_size;
229  int result = inflate(&m_zstream, Z_SYNC_FLUSH);
230 
231  if (result != Z_OK) {
232  m_buffer = nullptr;
233  m_buffer_size = 0;
234  }
235 
236  if (result != Z_OK && result != Z_STREAM_END) {
237  std::string message("gzip error: inflate failed: ");
238  if (m_zstream.msg) {
239  message.append(m_zstream.msg);
240  }
241  throw osmium::gzip_error(message, result);
242  }
243 
244  output.resize(static_cast<unsigned long>(m_zstream.next_out - reinterpret_cast<const unsigned char*>(output.data())));
245  }
246 
247  return output;
248  }
249 
250  void close() final {
251  inflateEnd(&m_zstream);
252  }
253 
254  }; // class GzipBufferDecompressor
255 
256  namespace detail {
257 
258  // we want the register_compression() function to run, setting
259  // the variable is only a side-effect, it will never be used
261  [](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); },
262  [](int fd) { return new osmium::io::GzipDecompressor(fd); },
263  [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
264  );
265 
266  // dummy function to silence the unused variable warning from above
267  inline bool get_registered_gzip_compression() noexcept {
268  return registered_gzip_compression;
269  }
270 
271  } // namespace detail
272 
273  } // namespace io
274 
275 } // namespace osmium
276 
277 #endif // OSMIUM_IO_GZIP_COMPRESSION_HPP
gzFile m_gzfile
Definition: gzip_compression.hpp:99
Definition: gzip_compression.hpp:189
gzip_error(const std::string &what, int error_code)
Definition: gzip_compression.hpp:69
gzFile m_gzfile
Definition: gzip_compression.hpp:147
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
Definition: gzip_compression.hpp:96
z_stream m_zstream
Definition: gzip_compression.hpp:193
static CompressionFactory & instance()
Definition: compression.hpp:151
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:91
GzipCompressor(int fd, fsync sync)
Definition: gzip_compression.hpp:103
size_t m_buffer_size
Definition: gzip_compression.hpp:192
Definition: gzip_compression.hpp:145
~GzipDecompressor() noexcept final
Definition: gzip_compression.hpp:159
Definition: compression.hpp:87
int m_fd
Definition: gzip_compression.hpp:98
GzipBufferDecompressor(const char *buffer, size_t size)
Definition: gzip_compression.hpp:197
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
const char * m_buffer
Definition: gzip_compression.hpp:191
void close() final
Definition: gzip_compression.hpp:250
fsync
Definition: writer_options.hpp:51
void close() final
Definition: gzip_compression.hpp:129
Definition: error.hpp:44
int system_errno
Definition: gzip_compression.hpp:67
Definition: gzip_compression.hpp:64
bool do_fsync() const
Definition: compression.hpp:68
std::string read() final
Definition: gzip_compression.hpp:167
int gzip_error_code
Definition: gzip_compression.hpp:66
GzipDecompressor(int fd)
Definition: gzip_compression.hpp:151
Definition: compression.hpp:62
void write(const std::string &data) final
Definition: gzip_compression.hpp:120
~GzipCompressor() noexcept final
Definition: gzip_compression.hpp:112
~GzipBufferDecompressor() noexcept final
Definition: gzip_compression.hpp:213
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:156
std::string read() final
Definition: gzip_compression.hpp:221
void close() final
Definition: gzip_compression.hpp:177