From c72de9990ae8f1744006d9c852023b882d5ed80c Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Mon, 13 Jul 2020 17:11:37 -0600 Subject: [PATCH] util: add CBufferedFile::SkipTo() to move ahead in the stream SkipTo() reads data from the file into the CBufferedFile object (memory), but, unlike this object's read() method, SkipTo() doesn't transfer data into a caller's memory buffer. This is useful because after skipping forward in the stream in this way, the user can, if needed, rewind the stream (SetPos()) and access the object's memory buffer including ranges that were skipped over (without needing to read from the disk file). --- src/streams.h | 48 ++++++++++++++++++--------- src/test/streams_tests.cpp | 67 +++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/src/streams.h b/src/streams.h index 0178df1c496..84b12f65aa6 100644 --- a/src/streams.h +++ b/src/streams.h @@ -612,7 +612,6 @@ private: uint64_t nRewind; //!< how many bytes we guarantee to rewind std::vector vchBuf; //!< the buffer -protected: //! read data from the source to fill the buffer bool Fill() { unsigned int pos = nSrcPos % vchBuf.size(); @@ -630,6 +629,28 @@ protected: return true; } + //! Advance the stream's read pointer (m_read_pos) by up to 'length' bytes, + //! filling the buffer from the file so that at least one byte is available. + //! Return a pointer to the available buffer data and the number of bytes + //! (which may be less than the requested length) that may be accessed + //! beginning at that pointer. + std::pair AdvanceStream(size_t length) + { + assert(m_read_pos <= nSrcPos); + if (m_read_pos + length > nReadLimit) { + throw std::ios_base::failure("Attempt to position past buffer limit"); + } + // If there are no bytes available, read from the file. + if (m_read_pos == nSrcPos && length > 0) Fill(); + + size_t buffer_offset{static_cast(m_read_pos % vchBuf.size())}; + size_t buffer_available{static_cast(vchBuf.size() - buffer_offset)}; + size_t bytes_until_source_pos{static_cast(nSrcPos - m_read_pos)}; + size_t advance{std::min({length, buffer_available, bytes_until_source_pos})}; + m_read_pos += advance; + return std::make_pair(&vchBuf[buffer_offset], advance); + } + public: CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), m_read_pos(0), nReadLimit(std::numeric_limits::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0}) @@ -667,24 +688,21 @@ public: //! read a number of bytes void read(Span dst) { - if (dst.size() + m_read_pos > nReadLimit) { - throw std::ios_base::failure("Read attempted past buffer limit"); - } while (dst.size() > 0) { - if (m_read_pos == nSrcPos) - Fill(); - unsigned int pos = m_read_pos % vchBuf.size(); - size_t nNow = dst.size(); - if (nNow + pos > vchBuf.size()) - nNow = vchBuf.size() - pos; - if (nNow + m_read_pos > nSrcPos) - nNow = nSrcPos - m_read_pos; - memcpy(dst.data(), &vchBuf[pos], nNow); - m_read_pos += nNow; - dst = dst.subspan(nNow); + auto [buffer_pointer, length]{AdvanceStream(dst.size())}; + memcpy(dst.data(), buffer_pointer, length); + dst = dst.subspan(length); } } + //! Move the read position ahead in the stream to the given position. + //! Use SetPos() to back up in the stream, not SkipTo(). + void SkipTo(const uint64_t file_pos) + { + assert(file_pos >= m_read_pos); + while (m_read_pos < file_pos) AdvanceStream(file_pos - m_read_pos); + } + //! return the current reading position uint64_t GetPos() const { return m_read_pos; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 0925e2e9ee4..b1b262eaded 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(false); } catch (const std::exception& e) { BOOST_CHECK(strstr(e.what(), - "Read attempted past buffer limit") != nullptr); + "Attempt to position past buffer limit") != nullptr); } // The default argument removes the limit completely. BOOST_CHECK(bf.SetLimit()); @@ -322,7 +322,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(!bf.SetPos(0)); // But we should now be positioned at least as far back as allowed // by the rewind window (relative to our farthest read position, 40). - BOOST_CHECK(bf.GetPos() <= 30); + BOOST_CHECK(bf.GetPos() <= 30U); // We can explicitly close the file, or the destructor will do it. bf.fclose(); @@ -330,6 +330,55 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) fs::remove(streams_test_filename); } +BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) +{ + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); + // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). + for (uint8_t j = 0; j < 40; ++j) { + fwrite(&j, 1, 1, file); + } + rewind(file); + + // The buffer is 25 bytes, allow rewinding 10 bytes. + CBufferedFile bf(file, 25, 10, 222, 333); + + uint8_t i; + // This is like bf >> (7-byte-variable), in that it will cause data + // to be read from the file into memory, but it's not copied to us. + bf.SkipTo(7); + BOOST_CHECK_EQUAL(bf.GetPos(), 7U); + bf >> i; + BOOST_CHECK_EQUAL(i, 7); + + // The bytes in the buffer up to offset 7 are valid and can be read. + BOOST_CHECK(bf.SetPos(0)); + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + bf >> i; + BOOST_CHECK_EQUAL(i, 1); + + bf.SkipTo(11); + bf >> i; + BOOST_CHECK_EQUAL(i, 11); + + // SkipTo() honors the transfer limit; we can't position beyond the limit. + bf.SetLimit(13); + try { + bf.SkipTo(14); + BOOST_CHECK(false); + } catch (const std::exception& e) { + BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr); + } + + // We can position exactly to the transfer limit. + bf.SkipTo(13); + BOOST_CHECK_EQUAL(bf.GetPos(), 13U); + + bf.fclose(); + fs::remove(streams_test_filename); +} + BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) { // Make this test deterministic. @@ -361,7 +410,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) // sizes; the boundaries of the objects can interact arbitrarily // with the CBufferFile's internal buffer. These first three // cases simulate objects of various sizes (1, 2, 5 bytes). - switch (InsecureRandRange(5)) { + switch (InsecureRandRange(6)) { case 0: { uint8_t a[1]; if (currentPos + 1 > fileSize) @@ -399,6 +448,16 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) break; } case 3: { + // SkipTo is similar to the "read" cases above, except + // we don't receive the data. + size_t skip_length{static_cast(InsecureRandRange(5))}; + if (currentPos + skip_length > fileSize) continue; + bf.SetLimit(currentPos + skip_length); + bf.SkipTo(currentPos + skip_length); + currentPos += skip_length; + break; + } + case 4: { // Find a byte value (that is at or ahead of the current position). size_t find = currentPos + InsecureRandRange(8); if (find >= fileSize) @@ -415,7 +474,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) currentPos++; break; } - case 4: { + case 5: { size_t requestPos = InsecureRandRange(maxPos + 4); bool okay = bf.SetPos(requestPos); // The new position may differ from the requested position