Libosmium  2.15.1
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  }; // class Compressor
86 
87  class Decompressor {
88 
89  std::atomic<std::size_t> m_file_size{0};
90  std::atomic<std::size_t> m_offset{0};
91 
92  public:
93 
94  enum {
95  input_buffer_size = 1024u * 1024u
96  };
97 
98  Decompressor() = default;
99 
100  Decompressor(const Decompressor&) = delete;
102 
105 
106  virtual ~Decompressor() noexcept = default;
107 
108  virtual std::string read() = 0;
109 
110  virtual void close() = 0;
111 
112  std::size_t file_size() const noexcept {
113  return m_file_size;
114  }
115 
116  void set_file_size(const std::size_t size) noexcept {
117  m_file_size = size;
118  }
119 
120  std::size_t offset() const noexcept {
121  return m_offset;
122  }
123 
124  void set_offset(const std::size_t offset) noexcept {
125  m_offset = offset;
126  }
127 
128  }; // class Decompressor
129 
138 
139  public:
140 
141  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
143  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
144 
145  private:
146 
150 
151  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
152 
154 
155  CompressionFactory() = default;
156 
158  const auto it = m_callbacks.find(compression);
159 
160  if (it != m_callbacks.end()) {
161  return it->second;
162  }
163 
164  std::string error_message{"Support for compression '"};
165  error_message += as_string(compression);
166  error_message += "' not compiled into this binary";
167  throw unsupported_file_format_error{error_message};
168  }
169 
170  public:
171 
174 
177 
178  ~CompressionFactory() noexcept = default;
179 
180  static CompressionFactory& instance() {
181  static CompressionFactory factory;
182  return factory;
183  }
184 
186  osmium::io::file_compression compression,
187  create_compressor_type create_compressor,
188  create_decompressor_type_fd create_decompressor_fd,
189  create_decompressor_type_buffer create_decompressor_buffer) {
190 
191  compression_map_type::value_type cc{compression,
192  std::make_tuple(create_compressor,
193  create_decompressor_fd,
194  create_decompressor_buffer)};
195 
196  return m_callbacks.insert(cc).second;
197  }
198 
199  template <typename... TArgs>
200  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
201  const auto callbacks = find_callbacks(compression);
202  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
203  }
204 
205  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
206  const auto callbacks = find_callbacks(compression);
207  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
208  p->set_file_size(osmium::file_size(fd));
209  return p;
210  }
211 
212  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
213  const auto callbacks = find_callbacks(compression);
214  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
215  }
216 
217  }; // class CompressionFactory
218 
219  class NoCompressor : public Compressor {
220 
221  int m_fd;
222 
223  public:
224 
225  NoCompressor(const int fd, const fsync sync) :
226  Compressor(sync),
227  m_fd(fd) {
228  }
229 
230  NoCompressor(const NoCompressor&) = delete;
232 
235 
236  ~NoCompressor() noexcept final {
237  try {
238  close();
239  } catch (...) {
240  // Ignore any exceptions because destructor must not throw.
241  }
242  }
243 
244  void write(const std::string& data) final {
245  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
246  }
247 
248  void close() final {
249  if (m_fd >= 0) {
250  const int fd = m_fd;
251  m_fd = -1;
252  if (do_fsync()) {
253  osmium::io::detail::reliable_fsync(fd);
254  }
255  osmium::io::detail::reliable_close(fd);
256  }
257  }
258 
259  }; // class NoCompressor
260 
261  class NoDecompressor : public Decompressor {
262 
263  int m_fd = -1;
264  const char* m_buffer = nullptr;
265  std::size_t m_buffer_size = 0;
266  std::size_t m_offset = 0;
267 
268  public:
269 
270  explicit NoDecompressor(const int fd) :
271  m_fd(fd) {
272  }
273 
274  NoDecompressor(const char* buffer, const std::size_t size) :
275  m_buffer(buffer),
276  m_buffer_size(size) {
277  }
278 
279  NoDecompressor(const NoDecompressor&) = delete;
281 
284 
285  ~NoDecompressor() noexcept final {
286  try {
287  close();
288  } catch (...) {
289  // Ignore any exceptions because destructor must not throw.
290  }
291  }
292 
293  std::string read() final {
294  std::string buffer;
295 
296  if (m_buffer) {
297  if (m_buffer_size != 0) {
298  const std::size_t size = m_buffer_size;
299  m_buffer_size = 0;
300  buffer.append(m_buffer, size);
301  }
302  } else {
304  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
305  buffer.resize(std::string::size_type(nread));
306  }
307 
308  m_offset += buffer.size();
309  set_offset(m_offset);
310 
311  return buffer;
312  }
313 
314  void close() final {
315  if (m_fd >= 0) {
316  const int fd = m_fd;
317  m_fd = -1;
318  osmium::io::detail::reliable_close(fd);
319  }
320  }
321 
322  }; // class NoDecompressor
323 
324  namespace detail {
325 
326  // we want the register_compression() function to run, setting
327  // the variable is only a side-effect, it will never be used
329  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
330  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
331  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
332  );
333 
334  // dummy function to silence the unused variable warning from above
335  inline bool get_registered_no_compression() noexcept {
336  return registered_no_compression;
337  }
338 
339  } // namespace detail
340 
341  } // namespace io
342 
343 } // namespace osmium
344 
345 #endif // OSMIUM_IO_COMPRESSION_HPP
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory(const CompressionFactory &)=delete
osmium::io::Decompressor::offset
std::size_t offset() const noexcept
Definition: compression.hpp:120
osmium::io::Compressor::operator=
Compressor & operator=(const Compressor &)=default
writer_options.hpp
osmium::io::Decompressor::Decompressor
Decompressor(Decompressor &&)=delete
osmium::io::NoCompressor::operator=
NoCompressor & operator=(const NoCompressor &)=delete
osmium::io::CompressionFactory::instance
static CompressionFactory & instance()
Definition: compression.hpp:180
osmium::io::Compressor::m_fsync
fsync m_fsync
Definition: compression.hpp:59
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory()=default
osmium::io::Compressor::close
virtual void close()=0
osmium::io::fsync::yes
@ yes
osmium::io::Compressor::Compressor
Compressor(Compressor &&) noexcept=default
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(NoDecompressor &&)=delete
osmium::io::CompressionFactory::create_decompressor_type_buffer
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:143
detail
Definition: attr.hpp:333
osmium::io::NoCompressor::~NoCompressor
~NoCompressor() noexcept final
Definition: compression.hpp:236
osmium::io::Compressor
Definition: compression.hpp:57
osmium::io::Decompressor
Definition: compression.hpp:87
osmium::io::NoDecompressor::close
void close() final
Definition: compression.hpp:314
osmium::io::NoDecompressor::operator=
NoDecompressor & operator=(const NoDecompressor &)=delete
osmium::io::NoCompressor::m_fd
int m_fd
Definition: compression.hpp:221
osmium::io::CompressionFactory::compression_map_type
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:151
osmium::io::CompressionFactory::create_decompressor
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:212
osmium::unsupported_file_format_error
Definition: error.hpp:56
osmium::io::Compressor::Compressor
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const NoDecompressor &)=delete
osmium::io::Decompressor::operator=
Decompressor & operator=(Decompressor &&)=delete
osmium::io::NoCompressor::NoCompressor
NoCompressor(NoCompressor &&)=delete
osmium::io::Compressor::write
virtual void write(const std::string &data)=0
osmium::io::NoDecompressor
Definition: compression.hpp:261
osmium::io::CompressionFactory::m_callbacks
compression_map_type m_callbacks
Definition: compression.hpp:153
osmium::io::NoDecompressor::read
std::string read() final
Definition: compression.hpp:293
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
osmium::io::Decompressor::input_buffer_size
@ input_buffer_size
Definition: compression.hpp:95
osmium::io::file_compression::none
@ none
osmium::io::NoCompressor
Definition: compression.hpp:219
osmium::io::NoCompressor::NoCompressor
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:225
osmium::io::CompressionFactory::create_compressor
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:200
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory(CompressionFactory &&)=delete
osmium::io::CompressionFactory::create_decompressor
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:205
osmium::util::file_size
std::size_t file_size(int fd)
Definition: file.hpp:109
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:274
osmium::io::NoCompressor::close
void close() final
Definition: compression.hpp:248
osmium::io::Decompressor::operator=
Decompressor & operator=(const Decompressor &)=delete
osmium::io::CompressionFactory::operator=
CompressionFactory & operator=(CompressionFactory &&)=delete
osmium::io::NoDecompressor::operator=
NoDecompressor & operator=(NoDecompressor &&)=delete
osmium::io::fsync
fsync
Definition: writer_options.hpp:51
osmium::io::CompressionFactory::callbacks_type
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:149
osmium::io::file_compression
file_compression
Definition: file_compression.hpp:42
osmium::io::Decompressor::~Decompressor
virtual ~Decompressor() noexcept=default
osmium::io::as_string
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
osmium::io::NoCompressor::NoCompressor
NoCompressor(const NoCompressor &)=delete
osmium::io::Compressor::Compressor
Compressor(const Compressor &)=default
osmium::io::CompressionFactory::create_decompressor_type_fd
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:142
osmium::io::CompressionFactory::register_compression
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:185
osmium::io::CompressionFactory::find_callbacks
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:157
std
Definition: location.hpp:550
osmium::io::CompressionFactory
Definition: compression.hpp:137
osmium::io::CompressionFactory::operator=
CompressionFactory & operator=(const CompressionFactory &)=delete
osmium::io::CompressionFactory::create_compressor_type
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:141
file.hpp
osmium::io::Decompressor::set_file_size
void set_file_size(const std::size_t size) noexcept
Definition: compression.hpp:116
file_compression.hpp
osmium::io::Decompressor::set_offset
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:124
osmium::io::CompressionFactory::~CompressionFactory
~CompressionFactory() noexcept=default
osmium::io::NoCompressor::operator=
NoCompressor & operator=(NoCompressor &&)=delete
osmium::io::Decompressor::Decompressor
Decompressor()=default
osmium::io::NoCompressor::write
void write(const std::string &data) final
Definition: compression.hpp:244
osmium::io::Compressor::do_fsync
bool do_fsync() const noexcept
Definition: compression.hpp:63
osmium::io::Decompressor::Decompressor
Decompressor(const Decompressor &)=delete
error.hpp
osmium::io::NoDecompressor::~NoDecompressor
~NoDecompressor() noexcept final
Definition: compression.hpp:285
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const int fd)
Definition: compression.hpp:270