zero_copy_stream_impl_lite.h 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. // Protocol Buffers - Google's data interchange format
  2. // Copyright 2008 Google Inc. All rights reserved.
  3. // https://developers.google.com/protocol-buffers/
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // Author: kenton@google.com (Kenton Varda)
  31. // Based on original Protocol Buffers design by
  32. // Sanjay Ghemawat, Jeff Dean, and others.
  33. //
  34. // This file contains common implementations of the interfaces defined in
  35. // zero_copy_stream.h which are included in the "lite" protobuf library.
  36. // These implementations cover I/O on raw arrays and strings, as well as
  37. // adaptors which make it easy to implement streams based on traditional
  38. // streams. Of course, many users will probably want to write their own
  39. // implementations of these interfaces specific to the particular I/O
  40. // abstractions they prefer to use, but these should cover the most common
  41. // cases.
  42. #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
  43. #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
  44. #include <iosfwd>
  45. #include <memory>
  46. #include <string>
  47. #include <google/protobuf/stubs/callback.h>
  48. #include <google/protobuf/stubs/common.h>
  49. #include <google/protobuf/io/zero_copy_stream.h>
  50. #include <google/protobuf/stubs/stl_util.h>
  51. #include <google/protobuf/port_def.inc>
  52. namespace google {
  53. namespace protobuf {
  54. namespace io {
  55. // ===================================================================
  56. // A ZeroCopyInputStream backed by an in-memory array of bytes.
  57. class PROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream {
  58. public:
  59. // Create an InputStream that returns the bytes pointed to by "data".
  60. // "data" remains the property of the caller but must remain valid until
  61. // the stream is destroyed. If a block_size is given, calls to Next()
  62. // will return data blocks no larger than the given size. Otherwise, the
  63. // first call to Next() returns the entire array. block_size is mainly
  64. // useful for testing; in production you would probably never want to set
  65. // it.
  66. ArrayInputStream(const void* data, int size, int block_size = -1);
  67. ~ArrayInputStream() override = default;
  68. // implements ZeroCopyInputStream ----------------------------------
  69. bool Next(const void** data, int* size) override;
  70. void BackUp(int count) override;
  71. bool Skip(int count) override;
  72. int64_t ByteCount() const override;
  73. private:
  74. const uint8_t* const data_; // The byte array.
  75. const int size_; // Total size of the array.
  76. const int block_size_; // How many bytes to return at a time.
  77. int position_;
  78. int last_returned_size_; // How many bytes we returned last time Next()
  79. // was called (used for error checking only).
  80. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream);
  81. };
  82. // ===================================================================
  83. // A ZeroCopyOutputStream backed by an in-memory array of bytes.
  84. class PROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream {
  85. public:
  86. // Create an OutputStream that writes to the bytes pointed to by "data".
  87. // "data" remains the property of the caller but must remain valid until
  88. // the stream is destroyed. If a block_size is given, calls to Next()
  89. // will return data blocks no larger than the given size. Otherwise, the
  90. // first call to Next() returns the entire array. block_size is mainly
  91. // useful for testing; in production you would probably never want to set
  92. // it.
  93. ArrayOutputStream(void* data, int size, int block_size = -1);
  94. ~ArrayOutputStream() override = default;
  95. // implements ZeroCopyOutputStream ---------------------------------
  96. bool Next(void** data, int* size) override;
  97. void BackUp(int count) override;
  98. int64_t ByteCount() const override;
  99. private:
  100. uint8_t* const data_; // The byte array.
  101. const int size_; // Total size of the array.
  102. const int block_size_; // How many bytes to return at a time.
  103. int position_;
  104. int last_returned_size_; // How many bytes we returned last time Next()
  105. // was called (used for error checking only).
  106. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream);
  107. };
  108. // ===================================================================
  109. // A ZeroCopyOutputStream which appends bytes to a string.
  110. class PROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream {
  111. public:
  112. // Create a StringOutputStream which appends bytes to the given string.
  113. // The string remains property of the caller, but it is mutated in arbitrary
  114. // ways and MUST NOT be accessed in any way until you're done with the
  115. // stream. Either be sure there's no further usage, or (safest) destroy the
  116. // stream before using the contents.
  117. //
  118. // Hint: If you call target->reserve(n) before creating the stream,
  119. // the first call to Next() will return at least n bytes of buffer
  120. // space.
  121. explicit StringOutputStream(std::string* target);
  122. ~StringOutputStream() override = default;
  123. // implements ZeroCopyOutputStream ---------------------------------
  124. bool Next(void** data, int* size) override;
  125. void BackUp(int count) override;
  126. int64_t ByteCount() const override;
  127. private:
  128. static constexpr size_t kMinimumSize = 16;
  129. std::string* target_;
  130. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream);
  131. };
  132. // Note: There is no StringInputStream. Instead, just create an
  133. // ArrayInputStream as follows:
  134. // ArrayInputStream input(str.data(), str.size());
  135. // ===================================================================
  136. // A generic traditional input stream interface.
  137. //
  138. // Lots of traditional input streams (e.g. file descriptors, C stdio
  139. // streams, and C++ iostreams) expose an interface where every read
  140. // involves copying bytes into a buffer. If you want to take such an
  141. // interface and make a ZeroCopyInputStream based on it, simply implement
  142. // CopyingInputStream and then use CopyingInputStreamAdaptor.
  143. //
  144. // CopyingInputStream implementations should avoid buffering if possible.
  145. // CopyingInputStreamAdaptor does its own buffering and will read data
  146. // in large blocks.
  147. class PROTOBUF_EXPORT CopyingInputStream {
  148. public:
  149. virtual ~CopyingInputStream() {}
  150. // Reads up to "size" bytes into the given buffer. Returns the number of
  151. // bytes read. Read() waits until at least one byte is available, or
  152. // returns zero if no bytes will ever become available (EOF), or -1 if a
  153. // permanent read error occurred.
  154. virtual int Read(void* buffer, int size) = 0;
  155. // Skips the next "count" bytes of input. Returns the number of bytes
  156. // actually skipped. This will always be exactly equal to "count" unless
  157. // EOF was reached or a permanent read error occurred.
  158. //
  159. // The default implementation just repeatedly calls Read() into a scratch
  160. // buffer.
  161. virtual int Skip(int count);
  162. };
  163. // A ZeroCopyInputStream which reads from a CopyingInputStream. This is
  164. // useful for implementing ZeroCopyInputStreams that read from traditional
  165. // streams. Note that this class is not really zero-copy.
  166. //
  167. // If you want to read from file descriptors or C++ istreams, this is
  168. // already implemented for you: use FileInputStream or IstreamInputStream
  169. // respectively.
  170. class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream {
  171. public:
  172. // Creates a stream that reads from the given CopyingInputStream.
  173. // If a block_size is given, it specifies the number of bytes that
  174. // should be read and returned with each call to Next(). Otherwise,
  175. // a reasonable default is used. The caller retains ownership of
  176. // copying_stream unless SetOwnsCopyingStream(true) is called.
  177. explicit CopyingInputStreamAdaptor(CopyingInputStream* copying_stream,
  178. int block_size = -1);
  179. ~CopyingInputStreamAdaptor() override;
  180. // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to
  181. // delete the underlying CopyingInputStream when it is destroyed.
  182. void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; }
  183. // implements ZeroCopyInputStream ----------------------------------
  184. bool Next(const void** data, int* size) override;
  185. void BackUp(int count) override;
  186. bool Skip(int count) override;
  187. int64_t ByteCount() const override;
  188. private:
  189. // Insures that buffer_ is not NULL.
  190. void AllocateBufferIfNeeded();
  191. // Frees the buffer and resets buffer_used_.
  192. void FreeBuffer();
  193. // The underlying copying stream.
  194. CopyingInputStream* copying_stream_;
  195. bool owns_copying_stream_;
  196. // True if we have seen a permanent error from the underlying stream.
  197. bool failed_;
  198. // The current position of copying_stream_, relative to the point where
  199. // we started reading.
  200. int64_t position_;
  201. // Data is read into this buffer. It may be NULL if no buffer is currently
  202. // in use. Otherwise, it points to an array of size buffer_size_.
  203. std::unique_ptr<uint8_t[]> buffer_;
  204. const int buffer_size_;
  205. // Number of valid bytes currently in the buffer (i.e. the size last
  206. // returned by Next()). 0 <= buffer_used_ <= buffer_size_.
  207. int buffer_used_;
  208. // Number of bytes in the buffer which were backed up over by a call to
  209. // BackUp(). These need to be returned again.
  210. // 0 <= backup_bytes_ <= buffer_used_
  211. int backup_bytes_;
  212. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingInputStreamAdaptor);
  213. };
  214. // ===================================================================
  215. // A generic traditional output stream interface.
  216. //
  217. // Lots of traditional output streams (e.g. file descriptors, C stdio
  218. // streams, and C++ iostreams) expose an interface where every write
  219. // involves copying bytes from a buffer. If you want to take such an
  220. // interface and make a ZeroCopyOutputStream based on it, simply implement
  221. // CopyingOutputStream and then use CopyingOutputStreamAdaptor.
  222. //
  223. // CopyingOutputStream implementations should avoid buffering if possible.
  224. // CopyingOutputStreamAdaptor does its own buffering and will write data
  225. // in large blocks.
  226. class PROTOBUF_EXPORT CopyingOutputStream {
  227. public:
  228. virtual ~CopyingOutputStream() {}
  229. // Writes "size" bytes from the given buffer to the output. Returns true
  230. // if successful, false on a write error.
  231. virtual bool Write(const void* buffer, int size) = 0;
  232. };
  233. // A ZeroCopyOutputStream which writes to a CopyingOutputStream. This is
  234. // useful for implementing ZeroCopyOutputStreams that write to traditional
  235. // streams. Note that this class is not really zero-copy.
  236. //
  237. // If you want to write to file descriptors or C++ ostreams, this is
  238. // already implemented for you: use FileOutputStream or OstreamOutputStream
  239. // respectively.
  240. class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream {
  241. public:
  242. // Creates a stream that writes to the given Unix file descriptor.
  243. // If a block_size is given, it specifies the size of the buffers
  244. // that should be returned by Next(). Otherwise, a reasonable default
  245. // is used.
  246. explicit CopyingOutputStreamAdaptor(CopyingOutputStream* copying_stream,
  247. int block_size = -1);
  248. ~CopyingOutputStreamAdaptor() override;
  249. // Writes all pending data to the underlying stream. Returns false if a
  250. // write error occurred on the underlying stream. (The underlying
  251. // stream itself is not necessarily flushed.)
  252. bool Flush();
  253. // Call SetOwnsCopyingStream(true) to tell the CopyingOutputStreamAdaptor to
  254. // delete the underlying CopyingOutputStream when it is destroyed.
  255. void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; }
  256. // implements ZeroCopyOutputStream ---------------------------------
  257. bool Next(void** data, int* size) override;
  258. void BackUp(int count) override;
  259. int64_t ByteCount() const override;
  260. bool WriteAliasedRaw(const void* data, int size) override;
  261. bool AllowsAliasing() const override { return true; }
  262. private:
  263. // Write the current buffer, if it is present.
  264. bool WriteBuffer();
  265. // Insures that buffer_ is not NULL.
  266. void AllocateBufferIfNeeded();
  267. // Frees the buffer.
  268. void FreeBuffer();
  269. // The underlying copying stream.
  270. CopyingOutputStream* copying_stream_;
  271. bool owns_copying_stream_;
  272. // True if we have seen a permanent error from the underlying stream.
  273. bool failed_;
  274. // The current position of copying_stream_, relative to the point where
  275. // we started writing.
  276. int64_t position_;
  277. // Data is written from this buffer. It may be NULL if no buffer is
  278. // currently in use. Otherwise, it points to an array of size buffer_size_.
  279. std::unique_ptr<uint8_t[]> buffer_;
  280. const int buffer_size_;
  281. // Number of valid bytes currently in the buffer (i.e. the size last
  282. // returned by Next()). When BackUp() is called, we just reduce this.
  283. // 0 <= buffer_used_ <= buffer_size_.
  284. int buffer_used_;
  285. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor);
  286. };
  287. // ===================================================================
  288. // A ZeroCopyInputStream which wraps some other stream and limits it to
  289. // a particular byte count.
  290. class PROTOBUF_EXPORT LimitingInputStream : public ZeroCopyInputStream {
  291. public:
  292. LimitingInputStream(ZeroCopyInputStream* input, int64_t limit);
  293. ~LimitingInputStream() override;
  294. // implements ZeroCopyInputStream ----------------------------------
  295. bool Next(const void** data, int* size) override;
  296. void BackUp(int count) override;
  297. bool Skip(int count) override;
  298. int64_t ByteCount() const override;
  299. private:
  300. ZeroCopyInputStream* input_;
  301. int64_t limit_; // Decreases as we go, becomes negative if we overshoot.
  302. int64_t prior_bytes_read_; // Bytes read on underlying stream at construction
  303. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream);
  304. };
  305. // ===================================================================
  306. // mutable_string_data() and as_string_data() are workarounds to improve
  307. // the performance of writing new data to an existing string. Unfortunately
  308. // the methods provided by the string class are suboptimal, and using memcpy()
  309. // is mildly annoying because it requires its pointer args to be non-NULL even
  310. // if we ask it to copy 0 bytes. Furthermore, string_as_array() has the
  311. // property that it always returns NULL if its arg is the empty string, exactly
  312. // what we want to avoid if we're using it in conjunction with memcpy()!
  313. // With C++11, the desired memcpy() boils down to memcpy(..., &(*s)[0], size),
  314. // where s is a string*. Without C++11, &(*s)[0] is not guaranteed to be safe,
  315. // so we use string_as_array(), and live with the extra logic that tests whether
  316. // *s is empty.
  317. // Return a pointer to mutable characters underlying the given string. The
  318. // return value is valid until the next time the string is resized. We
  319. // trust the caller to treat the return value as an array of length s->size().
  320. inline char* mutable_string_data(std::string* s) {
  321. // This should be simpler & faster than string_as_array() because the latter
  322. // is guaranteed to return NULL when *s is empty, so it has to check for that.
  323. return &(*s)[0];
  324. }
  325. // as_string_data(s) is equivalent to
  326. // ({ char* p = mutable_string_data(s); make_pair(p, p != NULL); })
  327. // Sometimes it's faster: in some scenarios p cannot be NULL, and then the
  328. // code can avoid that check.
  329. inline std::pair<char*, bool> as_string_data(std::string* s) {
  330. char* p = mutable_string_data(s);
  331. return std::make_pair(p, true);
  332. }
  333. } // namespace io
  334. } // namespace protobuf
  335. } // namespace google
  336. #include <google/protobuf/port_undef.inc>
  337. #endif // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__