streams_tests.cpp raw
1 // Copyright (c) 2012-present The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <flatfile.h>
6 #include <node/blockstorage.h>
7 #include <streams.h>
8 #include <test/util/common.h>
9 #include <test/util/random.h>
10 #include <test/util/setup_common.h>
11 #include <util/fs.h>
12 #include <util/obfuscation.h>
13 #include <util/strencodings.h>
14
15 #include <boost/test/unit_test.hpp>
16
17 using namespace std::string_literals;
18 using namespace util::hex_literals;
19
20 BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
21
22 // Check optimized obfuscation with random offsets and sizes to ensure proper
23 // handling of key wrapping. Also verify it roundtrips.
24 BOOST_AUTO_TEST_CASE(xor_random_chunks)
25 {
26 auto apply_random_xor_chunks{[&](std::span<std::byte> target, const Obfuscation& obfuscation) {
27 for (size_t offset{0}; offset < target.size();) {
28 const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)};
29 obfuscation(target.subspan(offset, chunk_size), offset);
30 offset += chunk_size;
31 }
32 }};
33
34 for (size_t test{0}; test < 100; ++test) {
35 const size_t write_size{1 + m_rng.randrange(100U)};
36 const std::vector original{m_rng.randbytes<std::byte>(write_size)};
37 std::vector roundtrip{original};
38
39 const auto key_bytes{m_rng.randbool() ? m_rng.randbytes<Obfuscation::KEY_SIZE>() : std::array<std::byte, Obfuscation::KEY_SIZE>{}};
40 const Obfuscation obfuscation{key_bytes};
41 apply_random_xor_chunks(roundtrip, obfuscation);
42 BOOST_CHECK_EQUAL(roundtrip.size(), original.size());
43 for (size_t i{0}; i < original.size(); ++i) {
44 BOOST_CHECK_EQUAL(roundtrip[i], original[i] ^ key_bytes[i % Obfuscation::KEY_SIZE]);
45 }
46
47 apply_random_xor_chunks(roundtrip, obfuscation);
48 BOOST_CHECK_EQUAL_COLLECTIONS(roundtrip.begin(), roundtrip.end(), original.begin(), original.end());
49 }
50 }
51
52 BOOST_AUTO_TEST_CASE(obfuscation_hexkey)
53 {
54 const auto key_bytes{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
55
56 const Obfuscation obfuscation{key_bytes};
57 BOOST_CHECK_EQUAL(obfuscation.HexKey(), HexStr(key_bytes));
58 }
59
60 BOOST_AUTO_TEST_CASE(obfuscation_serialize)
61 {
62 Obfuscation obfuscation{};
63 BOOST_CHECK(!obfuscation);
64
65 // Test loading a key.
66 std::vector key_in{m_rng.randbytes<std::byte>(Obfuscation::KEY_SIZE)};
67 DataStream ds_in;
68 ds_in << key_in;
69 BOOST_CHECK_EQUAL(ds_in.size(), 1 + Obfuscation::KEY_SIZE); // serialized as a vector
70 ds_in >> obfuscation;
71
72 // Test saving the key.
73 std::vector<std::byte> key_out;
74 DataStream ds_out;
75 ds_out << obfuscation;
76 ds_out >> key_out;
77
78 // Make sure saved key is the same.
79 BOOST_CHECK_EQUAL_COLLECTIONS(key_in.begin(), key_in.end(), key_out.begin(), key_out.end());
80 }
81
82 BOOST_AUTO_TEST_CASE(obfuscation_empty)
83 {
84 const Obfuscation null_obf{};
85 BOOST_CHECK(!null_obf);
86
87 const Obfuscation non_null_obf{"ff00ff00ff00ff00"_hex};
88 BOOST_CHECK(non_null_obf);
89 }
90
91 BOOST_AUTO_TEST_CASE(streams_scoped_data_stream_usage)
92 {
93 DataStream stream{};
94 {
95 ScopedDataStreamUsage usage{stream};
96 stream << uint8_t{42};
97 BOOST_CHECK_GT(stream.size(), 0U);
98 }
99 BOOST_CHECK(stream.empty());
100
101 {
102 ScopedDataStreamUsage usage{stream};
103 stream << uint16_t{42};
104 BOOST_CHECK_GT(stream.size(), 0U);
105 }
106 BOOST_CHECK(stream.empty());
107 }
108
109 BOOST_AUTO_TEST_CASE(xor_file)
110 {
111 fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
112 auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
113 const std::vector<uint8_t> test1{1, 2, 3};
114 const std::vector<uint8_t> test2{4, 5};
115 const Obfuscation obfuscation{"ff00ff00ff00ff00"_hex};
116
117 {
118 // Check errors for missing file
119 AutoFile xor_file{raw_file("rb"), obfuscation};
120 BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullptr"});
121 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullptr"});
122 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullptr"});
123 BOOST_CHECK_EXCEPTION(xor_file.size(), std::ios_base::failure, HasReason{"AutoFile::size: file handle is nullptr"});
124 }
125 {
126 #ifdef __MINGW64__
127 // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210
128 const char* mode = "wb";
129 #else
130 const char* mode = "wbx";
131 #endif
132 AutoFile xor_file{raw_file(mode), obfuscation};
133 xor_file << test1 << test2;
134 BOOST_CHECK_EQUAL(xor_file.size(), 7);
135 BOOST_REQUIRE_EQUAL(xor_file.fclose(), 0);
136 }
137 {
138 // Read raw from disk
139 AutoFile non_xor_file{raw_file("rb")};
140 std::vector<std::byte> raw(7);
141 non_xor_file >> std::span{raw};
142 BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa");
143 // Check that no padding exists
144 BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
145 BOOST_CHECK_EQUAL(non_xor_file.size(), 7);
146 }
147 {
148 AutoFile xor_file{raw_file("rb"), obfuscation};
149 std::vector<std::byte> read1, read2;
150 xor_file >> read1 >> read2;
151 BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
152 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2));
153 // Check that eof was reached
154 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
155 BOOST_CHECK_EQUAL(xor_file.size(), 7);
156 }
157 {
158 AutoFile xor_file{raw_file("rb"), obfuscation};
159 std::vector<std::byte> read2;
160 // Check that ignore works
161 xor_file.ignore(4);
162 xor_file >> read2;
163 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2));
164 // Check that ignore and read fail now
165 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
166 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
167 BOOST_CHECK_EQUAL(xor_file.size(), 7);
168 }
169 }
170
171 BOOST_AUTO_TEST_CASE(streams_vector_writer)
172 {
173 unsigned char a(1);
174 unsigned char b(2);
175 unsigned char bytes[] = {3, 4, 5, 6};
176 std::vector<unsigned char> vch;
177
178 // Each test runs twice. Serializing a second time at the same starting
179 // point should yield the same results, even if the first test grew the
180 // vector.
181
182 VectorWriter{vch, 0, a, b};
183 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}}));
184 VectorWriter{vch, 0, a, b};
185 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}}));
186 vch.clear();
187
188 VectorWriter{vch, 2, a, b};
189 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}}));
190 VectorWriter{vch, 2, a, b};
191 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}}));
192 vch.clear();
193
194 vch.resize(5, 0);
195 VectorWriter{vch, 2, a, b};
196 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}}));
197 VectorWriter{vch, 2, a, b};
198 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}}));
199 vch.clear();
200
201 vch.resize(4, 0);
202 VectorWriter{vch, 3, a, b};
203 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}}));
204 VectorWriter{vch, 3, a, b};
205 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}}));
206 vch.clear();
207
208 vch.resize(4, 0);
209 VectorWriter{vch, 4, a, b};
210 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}}));
211 VectorWriter{vch, 4, a, b};
212 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}}));
213 vch.clear();
214
215 VectorWriter{vch, 0, bytes};
216 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}}));
217 VectorWriter{vch, 0, bytes};
218 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}}));
219 vch.clear();
220
221 vch.resize(4, 8);
222 VectorWriter{vch, 2, a, bytes, b};
223 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}}));
224 VectorWriter{vch, 2, a, bytes, b};
225 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}}));
226 vch.clear();
227 }
228
229 BOOST_AUTO_TEST_CASE(streams_span_writer)
230 {
231 unsigned char a(1);
232 unsigned char b(2);
233 unsigned char bytes[] = {3, 4, 5, 6};
234 std::array<std::byte, 8> arr{};
235
236 // Test operator<<
237 SpanWriter writer{arr};
238 writer << a << b;
239 BOOST_CHECK_EQUAL(HexStr(arr), "0102000000000000");
240
241 // Use variadic constructor and write to subspan.
242 SpanWriter{std::span{arr}.subspan(2), a, bytes, b};
243 BOOST_CHECK_EQUAL(HexStr(arr), "0102010304050602");
244
245 // Writing past the end throws
246 std::array<std::byte, 1> small{};
247 BOOST_CHECK_THROW(SpanWriter(std::span{small}, a, b), std::ios_base::failure);
248 BOOST_CHECK_THROW(SpanWriter(std::span{small}) << a << b, std::ios_base::failure);
249 }
250
251 BOOST_AUTO_TEST_CASE(streams_vector_reader)
252 {
253 std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6};
254
255 SpanReader reader{vch};
256 BOOST_CHECK_EQUAL(reader.size(), 6U);
257 BOOST_CHECK(!reader.empty());
258
259 // Read a single byte as an unsigned char.
260 unsigned char a;
261 reader >> a;
262 BOOST_CHECK_EQUAL(a, 1);
263 BOOST_CHECK_EQUAL(reader.size(), 5U);
264 BOOST_CHECK(!reader.empty());
265
266 // Read a single byte as a int8_t.
267 int8_t b;
268 reader >> b;
269 BOOST_CHECK_EQUAL(b, -1);
270 BOOST_CHECK_EQUAL(reader.size(), 4U);
271 BOOST_CHECK(!reader.empty());
272
273 // Read a 4 bytes as an unsigned int.
274 unsigned int c;
275 reader >> c;
276 BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256
277 BOOST_CHECK_EQUAL(reader.size(), 0U);
278 BOOST_CHECK(reader.empty());
279
280 // Reading after end of byte vector throws an error.
281 signed int d;
282 BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
283
284 // Read a 4 bytes as a signed int from the beginning of the buffer.
285 SpanReader new_reader{vch};
286 new_reader >> d;
287 BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256
288 BOOST_CHECK_EQUAL(new_reader.size(), 2U);
289 BOOST_CHECK(!new_reader.empty());
290
291 // Reading after end of byte vector throws an error even if the reader is
292 // not totally empty.
293 BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure);
294 }
295
296 BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue)
297 {
298 std::vector<uint8_t> data{0x82, 0xa7, 0x31};
299 SpanReader reader{data};
300 uint32_t varint = 0;
301 // Deserialize into r-value
302 reader >> VARINT(varint);
303 BOOST_CHECK_EQUAL(varint, 54321U);
304 BOOST_CHECK(reader.empty());
305 }
306
307 BOOST_AUTO_TEST_CASE(bitstream_reader_writer)
308 {
309 DataStream data{};
310
311 BitStreamWriter bit_writer{data};
312 bit_writer.Write(0, 1);
313 bit_writer.Write(2, 2);
314 bit_writer.Write(6, 3);
315 bit_writer.Write(11, 4);
316 bit_writer.Write(1, 5);
317 bit_writer.Write(32, 6);
318 bit_writer.Write(7, 7);
319 bit_writer.Write(30497, 16);
320 bit_writer.Flush();
321
322 DataStream data_copy{data};
323 uint32_t serialized_int1;
324 data >> serialized_int1;
325 BOOST_CHECK_EQUAL(serialized_int1, uint32_t{0x7700C35A}); // NOTE: Serialized as LE
326 uint16_t serialized_int2;
327 data >> serialized_int2;
328 BOOST_CHECK_EQUAL(serialized_int2, uint16_t{0x1072}); // NOTE: Serialized as LE
329
330 BitStreamReader bit_reader{data_copy};
331 BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U);
332 BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U);
333 BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U);
334 BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U);
335 BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U);
336 BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U);
337 BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U);
338 BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U);
339 BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure);
340 }
341
342 BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
343 {
344 // Degenerate case
345 {
346 DataStream ds{};
347 Obfuscation{}(ds);
348 BOOST_CHECK_EQUAL(""s, ds.str());
349 }
350
351 {
352 const Obfuscation obfuscation{"ffffffffffffffff"_hex};
353
354 DataStream ds{"0ff0"_hex};
355 obfuscation(ds);
356 BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
357 }
358
359 {
360 const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex};
361
362 DataStream ds{"f00f"_hex};
363 obfuscation(ds);
364 BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
365 }
366 }
367
368 BOOST_AUTO_TEST_CASE(streams_buffered_file)
369 {
370 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
371 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
372
373 // The value at each offset is the offset.
374 for (uint8_t j = 0; j < 40; ++j) {
375 file << j;
376 }
377 file.seek(0, SEEK_SET);
378
379 // The buffer size (second arg) must be greater than the rewind
380 // amount (third arg).
381 try {
382 BufferedFile bfbad{file, 25, 25};
383 BOOST_CHECK(false);
384 } catch (const std::exception& e) {
385 BOOST_CHECK(strstr(e.what(),
386 "Rewind limit must be less than buffer size") != nullptr);
387 }
388
389 // The buffer is 25 bytes, allow rewinding 10 bytes.
390 BufferedFile bf{file, 25, 10};
391 BOOST_CHECK(!bf.eof());
392
393 uint8_t i;
394 bf >> i;
395 BOOST_CHECK_EQUAL(i, 0);
396 bf >> i;
397 BOOST_CHECK_EQUAL(i, 1);
398
399 // After reading bytes 0 and 1, we're positioned at 2.
400 BOOST_CHECK_EQUAL(bf.GetPos(), 2U);
401
402 // Rewind to offset 0, ok (within the 10 byte window).
403 BOOST_CHECK(bf.SetPos(0));
404 bf >> i;
405 BOOST_CHECK_EQUAL(i, 0);
406
407 // We can go forward to where we've been, but beyond may fail.
408 BOOST_CHECK(bf.SetPos(2));
409 bf >> i;
410 BOOST_CHECK_EQUAL(i, 2);
411
412 // If you know the maximum number of bytes that should be
413 // read to deserialize the variable, you can limit the read
414 // extent. The current file offset is 3, so the following
415 // SetLimit() allows zero bytes to be read.
416 BOOST_CHECK(bf.SetLimit(3));
417 try {
418 bf >> i;
419 BOOST_CHECK(false);
420 } catch (const std::exception& e) {
421 BOOST_CHECK(strstr(e.what(),
422 "Attempt to position past buffer limit") != nullptr);
423 }
424 // The default argument removes the limit completely.
425 BOOST_CHECK(bf.SetLimit());
426 // The read position should still be at 3 (no change).
427 BOOST_CHECK_EQUAL(bf.GetPos(), 3U);
428
429 // Read from current offset, 3, forward until position 10.
430 for (uint8_t j = 3; j < 10; ++j) {
431 bf >> i;
432 BOOST_CHECK_EQUAL(i, j);
433 }
434 BOOST_CHECK_EQUAL(bf.GetPos(), 10U);
435
436 // We're guaranteed (just barely) to be able to rewind to zero.
437 BOOST_CHECK(bf.SetPos(0));
438 BOOST_CHECK_EQUAL(bf.GetPos(), 0U);
439 bf >> i;
440 BOOST_CHECK_EQUAL(i, 0);
441
442 // We can set the position forward again up to the farthest
443 // into the stream we've been, but no farther. (Attempting
444 // to go farther may succeed, but it's not guaranteed.)
445 BOOST_CHECK(bf.SetPos(10));
446 bf >> i;
447 BOOST_CHECK_EQUAL(i, 10);
448 BOOST_CHECK_EQUAL(bf.GetPos(), 11U);
449
450 // Now it's only guaranteed that we can rewind to offset 1
451 // (current read position, 11, minus rewind amount, 10).
452 BOOST_CHECK(bf.SetPos(1));
453 BOOST_CHECK_EQUAL(bf.GetPos(), 1U);
454 bf >> i;
455 BOOST_CHECK_EQUAL(i, 1);
456
457 // We can stream into large variables, even larger than
458 // the buffer size.
459 BOOST_CHECK(bf.SetPos(11));
460 {
461 uint8_t a[40 - 11];
462 bf >> a;
463 for (uint8_t j = 0; j < sizeof(a); ++j) {
464 BOOST_CHECK_EQUAL(a[j], 11 + j);
465 }
466 }
467 BOOST_CHECK_EQUAL(bf.GetPos(), 40U);
468
469 // We've read the entire file, the next read should throw.
470 try {
471 bf >> i;
472 BOOST_CHECK(false);
473 } catch (const std::exception& e) {
474 BOOST_CHECK(strstr(e.what(),
475 "BufferedFile::Fill: end of file") != nullptr);
476 }
477 // Attempting to read beyond the end sets the EOF indicator.
478 BOOST_CHECK(bf.eof());
479
480 // Still at offset 40, we can go back 10, to 30.
481 BOOST_CHECK_EQUAL(bf.GetPos(), 40U);
482 BOOST_CHECK(bf.SetPos(30));
483 bf >> i;
484 BOOST_CHECK_EQUAL(i, 30);
485 BOOST_CHECK_EQUAL(bf.GetPos(), 31U);
486
487 // We're too far to rewind to position zero.
488 BOOST_CHECK(!bf.SetPos(0));
489 // But we should now be positioned at least as far back as allowed
490 // by the rewind window (relative to our farthest read position, 40).
491 BOOST_CHECK(bf.GetPos() <= 30U);
492
493 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
494
495 fs::remove(streams_test_filename);
496 }
497
498 BOOST_AUTO_TEST_CASE(streams_buffered_file_skip)
499 {
500 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
501 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
502 // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01).
503 for (uint8_t j = 0; j < 40; ++j) {
504 file << j;
505 }
506 file.seek(0, SEEK_SET);
507
508 // The buffer is 25 bytes, allow rewinding 10 bytes.
509 BufferedFile bf{file, 25, 10};
510
511 uint8_t i;
512 // This is like bf >> (7-byte-variable), in that it will cause data
513 // to be read from the file into memory, but it's not copied to us.
514 bf.SkipTo(7);
515 BOOST_CHECK_EQUAL(bf.GetPos(), 7U);
516 bf >> i;
517 BOOST_CHECK_EQUAL(i, 7);
518
519 // The bytes in the buffer up to offset 7 are valid and can be read.
520 BOOST_CHECK(bf.SetPos(0));
521 bf >> i;
522 BOOST_CHECK_EQUAL(i, 0);
523 bf >> i;
524 BOOST_CHECK_EQUAL(i, 1);
525
526 bf.SkipTo(11);
527 bf >> i;
528 BOOST_CHECK_EQUAL(i, 11);
529
530 // SkipTo() honors the transfer limit; we can't position beyond the limit.
531 bf.SetLimit(13);
532 try {
533 bf.SkipTo(14);
534 BOOST_CHECK(false);
535 } catch (const std::exception& e) {
536 BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr);
537 }
538
539 // We can position exactly to the transfer limit.
540 bf.SkipTo(13);
541 BOOST_CHECK_EQUAL(bf.GetPos(), 13U);
542
543 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
544 fs::remove(streams_test_filename);
545 }
546
547 BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
548 {
549 // Make this test deterministic.
550 SeedRandomForTest(SeedRand::ZEROS);
551
552 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
553 for (int rep = 0; rep < 50; ++rep) {
554 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
555 size_t fileSize = m_rng.randrange(256);
556 for (uint8_t i = 0; i < fileSize; ++i) {
557 file << i;
558 }
559 file.seek(0, SEEK_SET);
560
561 size_t bufSize = m_rng.randrange(300) + 1;
562 size_t rewindSize = m_rng.randrange(bufSize);
563 BufferedFile bf{file, bufSize, rewindSize};
564 size_t currentPos = 0;
565 size_t maxPos = 0;
566 for (int step = 0; step < 100; ++step) {
567 if (currentPos >= fileSize)
568 break;
569
570 // We haven't read to the end of the file yet.
571 BOOST_CHECK(!bf.eof());
572 BOOST_CHECK_EQUAL(bf.GetPos(), currentPos);
573
574 // Pretend the file consists of a series of objects of varying
575 // sizes; the boundaries of the objects can interact arbitrarily
576 // with the CBufferFile's internal buffer. These first three
577 // cases simulate objects of various sizes (1, 2, 5 bytes).
578 switch (m_rng.randrange(6)) {
579 case 0: {
580 uint8_t a[1];
581 if (currentPos + 1 > fileSize)
582 continue;
583 bf.SetLimit(currentPos + 1);
584 bf >> a;
585 for (uint8_t i = 0; i < 1; ++i) {
586 BOOST_CHECK_EQUAL(a[i], currentPos);
587 currentPos++;
588 }
589 break;
590 }
591 case 1: {
592 uint8_t a[2];
593 if (currentPos + 2 > fileSize)
594 continue;
595 bf.SetLimit(currentPos + 2);
596 bf >> a;
597 for (uint8_t i = 0; i < 2; ++i) {
598 BOOST_CHECK_EQUAL(a[i], currentPos);
599 currentPos++;
600 }
601 break;
602 }
603 case 2: {
604 uint8_t a[5];
605 if (currentPos + 5 > fileSize)
606 continue;
607 bf.SetLimit(currentPos + 5);
608 bf >> a;
609 for (uint8_t i = 0; i < 5; ++i) {
610 BOOST_CHECK_EQUAL(a[i], currentPos);
611 currentPos++;
612 }
613 break;
614 }
615 case 3: {
616 // SkipTo is similar to the "read" cases above, except
617 // we don't receive the data.
618 size_t skip_length{static_cast<size_t>(m_rng.randrange(5))};
619 if (currentPos + skip_length > fileSize) continue;
620 bf.SetLimit(currentPos + skip_length);
621 bf.SkipTo(currentPos + skip_length);
622 currentPos += skip_length;
623 break;
624 }
625 case 4: {
626 // Find a byte value (that is at or ahead of the current position).
627 size_t find = currentPos + m_rng.randrange(8);
628 if (find >= fileSize)
629 find = fileSize - 1;
630 bf.FindByte(std::byte(find));
631 // The value at each offset is the offset.
632 BOOST_CHECK_EQUAL(bf.GetPos(), find);
633 currentPos = find;
634
635 bf.SetLimit(currentPos + 1);
636 uint8_t i;
637 bf >> i;
638 BOOST_CHECK_EQUAL(i, currentPos);
639 currentPos++;
640 break;
641 }
642 case 5: {
643 size_t requestPos = m_rng.randrange(maxPos + 4);
644 bool okay = bf.SetPos(requestPos);
645 // The new position may differ from the requested position
646 // because we may not be able to rewind beyond the rewind
647 // window, and we may not be able to move forward beyond the
648 // farthest position we've reached so far.
649 currentPos = bf.GetPos();
650 BOOST_CHECK_EQUAL(okay, currentPos == requestPos);
651 // Check that we can position within the rewind window.
652 if (requestPos <= maxPos &&
653 maxPos > rewindSize &&
654 requestPos >= maxPos - rewindSize) {
655 // We requested a position within the rewind window.
656 BOOST_CHECK(okay);
657 }
658 break;
659 }
660 }
661 if (maxPos < currentPos)
662 maxPos = currentPos;
663 }
664 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
665 }
666 fs::remove(streams_test_filename);
667 }
668
669 BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content)
670 {
671 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)};
672 const size_t buf_size{1 + m_rng.randrange(file_size)};
673 const FlatFilePos pos{0, 0};
674
675 const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE};
676 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
677
678 // Write out the file with random content
679 {
680 AutoFile f{test_file.Open(pos, /*read_only=*/false), obfuscation};
681 f.write(m_rng.randbytes<std::byte>(file_size));
682 BOOST_REQUIRE_EQUAL(f.fclose(), 0);
683 }
684 BOOST_CHECK_EQUAL(fs::file_size(test_file.FileName(pos)), file_size);
685
686 {
687 AutoFile direct_file{test_file.Open(pos, /*read_only=*/true), obfuscation};
688
689 AutoFile buffered_file{test_file.Open(pos, /*read_only=*/true), obfuscation};
690 BufferedReader buffered_reader{std::move(buffered_file), buf_size};
691
692 for (size_t total_read{0}; total_read < file_size;) {
693 const size_t read{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_read))};
694
695 DataBuffer direct_file_buffer{read};
696 direct_file.read(direct_file_buffer);
697
698 DataBuffer buffered_buffer{read};
699 buffered_reader.read(buffered_buffer);
700
701 BOOST_CHECK_EQUAL_COLLECTIONS(
702 direct_file_buffer.begin(), direct_file_buffer.end(),
703 buffered_buffer.begin(), buffered_buffer.end()
704 );
705
706 total_read += read;
707 }
708
709 {
710 DataBuffer excess_byte{1};
711 BOOST_CHECK_EXCEPTION(direct_file.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
712 }
713
714 {
715 DataBuffer excess_byte{1};
716 BOOST_CHECK_EXCEPTION(buffered_reader.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
717 }
718 }
719
720 fs::remove(test_file.FileName(pos));
721 }
722
723 BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content)
724 {
725 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)};
726 const size_t buf_size{1 + m_rng.randrange(file_size)};
727 const FlatFilePos pos{0, 0};
728
729 const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE};
730 const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE};
731 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
732
733 {
734 DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)};
735
736 AutoFile direct_file{test_direct.Open(pos, /*read_only=*/false), obfuscation};
737
738 AutoFile buffered_file{test_buffered.Open(pos, /*read_only=*/false), obfuscation};
739 {
740 BufferedWriter buffered{buffered_file, buf_size};
741
742 for (size_t total_written{0}; total_written < file_size;) {
743 const size_t write_size{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_written))};
744
745 auto current_span = std::span{test_data}.subspan(total_written, write_size);
746 direct_file.write(current_span);
747 buffered.write(current_span);
748
749 total_written += write_size;
750 }
751 }
752 BOOST_REQUIRE_EQUAL(buffered_file.fclose(), 0);
753 BOOST_REQUIRE_EQUAL(direct_file.fclose(), 0);
754 }
755
756 // Compare the resulting files
757 DataBuffer direct_result{file_size};
758 {
759 AutoFile verify_direct{test_direct.Open(pos, /*read_only=*/true), obfuscation};
760 verify_direct.read(direct_result);
761
762 DataBuffer excess_byte{1};
763 BOOST_CHECK_EXCEPTION(verify_direct.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
764 }
765
766 DataBuffer buffered_result{file_size};
767 {
768 AutoFile verify_buffered{test_buffered.Open(pos, /*read_only=*/true), obfuscation};
769 verify_buffered.read(buffered_result);
770
771 DataBuffer excess_byte{1};
772 BOOST_CHECK_EXCEPTION(verify_buffered.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
773 }
774
775 BOOST_CHECK_EQUAL_COLLECTIONS(
776 direct_result.begin(), direct_result.end(),
777 buffered_result.begin(), buffered_result.end()
778 );
779
780 fs::remove(test_direct.FileName(pos));
781 fs::remove(test_buffered.FileName(pos));
782 }
783
784 BOOST_AUTO_TEST_CASE(buffered_writer_reader)
785 {
786 const uint32_t v1{m_rng.rand32()}, v2{m_rng.rand32()}, v3{m_rng.rand32()};
787 const fs::path test_file{m_args.GetDataDirBase() / "test_buffered_write_read.bin"};
788
789 // Write out the values through a precisely sized BufferedWriter
790 AutoFile file{fsbridge::fopen(test_file, "w+b")};
791 {
792 BufferedWriter f(file, sizeof(v1) + sizeof(v2) + sizeof(v3));
793 f << v1 << v2;
794 f.write(std::as_bytes(std::span{&v3, 1}));
795 }
796 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
797
798 // Read back and verify using BufferedReader
799 {
800 uint32_t _v1{0}, _v2{0}, _v3{0};
801 AutoFile file{fsbridge::fopen(test_file, "rb")};
802 BufferedReader f(std::move(file), sizeof(v1) + sizeof(v2) + sizeof(v3));
803 f >> _v1 >> _v2;
804 f.read(std::as_writable_bytes(std::span{&_v3, 1}));
805 BOOST_CHECK_EQUAL(_v1, v1);
806 BOOST_CHECK_EQUAL(_v2, v2);
807 BOOST_CHECK_EQUAL(_v3, v3);
808
809 DataBuffer excess_byte{1};
810 BOOST_CHECK_EXCEPTION(f.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
811 }
812
813 fs::remove(test_file);
814 }
815
816 BOOST_AUTO_TEST_CASE(streams_hashed)
817 {
818 DataStream stream{};
819 HashedSourceWriter hash_writer{stream};
820 const std::string data{"bitcoin"};
821 hash_writer << data;
822
823 HashVerifier hash_verifier{stream};
824 std::string result;
825 hash_verifier >> result;
826 BOOST_CHECK_EQUAL(data, result);
827 BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash());
828 }
829
830 BOOST_AUTO_TEST_CASE(size_preserves_position)
831 {
832 const fs::path path = m_args.GetDataDirBase() / "size_pos_test.bin";
833 AutoFile f{fsbridge::fopen(path, "w+b")};
834 for (uint8_t j = 0; j < 10; ++j) {
835 f << j;
836 }
837
838 // Test that usage of size() does not change the current position
839 //
840 // Case: Pos at beginning of the file
841 f.seek(0, SEEK_SET);
842 (void)f.size();
843 uint8_t first{};
844 f >> first;
845 BOOST_CHECK_EQUAL(first, 0);
846
847 // Case: Pos at middle of the file
848 f.seek(0, SEEK_SET);
849 // Move pos to middle
850 f.ignore(4);
851 (void)f.size();
852 uint8_t middle{};
853 f >> middle;
854 // Pos still at 4
855 BOOST_CHECK_EQUAL(middle, 4);
856
857 // Case: Pos at EOF
858 f.seek(0, SEEK_END);
859 (void)f.size();
860 uint8_t end{};
861 BOOST_CHECK_EXCEPTION(f >> end, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
862
863 BOOST_REQUIRE_EQUAL(f.fclose(), 0);
864 fs::remove(path);
865 }
866
867 BOOST_AUTO_TEST_SUITE_END()
868