streams_tests.cpp raw
1 // Copyright (c) 2012-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <flatfile.h>
6 #include <node/blockstorage.h>
7 #include <streams.h>
8 #include <test/util/random.h>
9 #include <test/util/setup_common.h>
10 #include <util/fs.h>
11 #include <util/obfuscation.h>
12 #include <util/strencodings.h>
13
14 #include <boost/test/unit_test.hpp>
15
16 using namespace std::string_literals;
17 using namespace util::hex_literals;
18
19 BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
20
21 // Test that obfuscation can be properly reverted even with random chunk sizes.
22 BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
23 {
24 auto apply_random_xor_chunks{[&](std::span<std::byte> target, const Obfuscation& obfuscation) {
25 for (size_t offset{0}; offset < target.size();) {
26 const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)};
27 obfuscation(target.subspan(offset, chunk_size), offset);
28 offset += chunk_size;
29 }
30 }};
31
32 for (size_t test{0}; test < 100; ++test) {
33 const size_t write_size{1 + m_rng.randrange(100U)};
34 const std::vector original{m_rng.randbytes<std::byte>(write_size)};
35 std::vector roundtrip{original};
36
37 const auto key_bytes{m_rng.randbool() ? m_rng.randbytes<Obfuscation::KEY_SIZE>() : std::array<std::byte, Obfuscation::KEY_SIZE>{}};
38 const Obfuscation obfuscation{key_bytes};
39 apply_random_xor_chunks(roundtrip, obfuscation);
40
41 const bool key_all_zeros{std::ranges::all_of(
42 std::span{key_bytes}.first(std::min(write_size, Obfuscation::KEY_SIZE)), [](auto b) { return b == std::byte{0}; })};
43 BOOST_CHECK(key_all_zeros ? original == roundtrip : original != roundtrip);
44
45 apply_random_xor_chunks(roundtrip, obfuscation);
46 BOOST_CHECK(original == roundtrip);
47 }
48 }
49
50 // Compares optimized obfuscation against a trivial, byte-by-byte reference implementation
51 // with random offsets to ensure proper handling of key wrapping.
52 BOOST_AUTO_TEST_CASE(xor_bytes_reference)
53 {
54 auto expected_xor{[](std::span<std::byte> target, std::span<const std::byte, Obfuscation::KEY_SIZE> obfuscation, size_t key_offset) {
55 for (auto& b : target) {
56 b ^= obfuscation[key_offset++ % obfuscation.size()];
57 }
58 }};
59
60 for (size_t test{0}; test < 100; ++test) {
61 const size_t write_size{1 + m_rng.randrange(100U)};
62 const size_t key_offset{m_rng.randrange(3 * Obfuscation::KEY_SIZE)}; // Make sure the key can wrap around
63 const size_t write_offset{std::min(write_size, m_rng.randrange(Obfuscation::KEY_SIZE * 2))}; // Write unaligned data
64
65 const auto key_bytes{m_rng.randbool() ? m_rng.randbytes<Obfuscation::KEY_SIZE>() : std::array<std::byte, Obfuscation::KEY_SIZE>{}};
66 const Obfuscation obfuscation{key_bytes};
67 std::vector expected{m_rng.randbytes<std::byte>(write_size)};
68 std::vector actual{expected};
69
70 expected_xor(std::span{expected}.subspan(write_offset), key_bytes, key_offset);
71 obfuscation(std::span{actual}.subspan(write_offset), key_offset);
72
73 BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
74 }
75 }
76
77 BOOST_AUTO_TEST_CASE(obfuscation_hexkey)
78 {
79 const auto key_bytes{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
80
81 const Obfuscation obfuscation{key_bytes};
82 BOOST_CHECK_EQUAL(obfuscation.HexKey(), HexStr(key_bytes));
83 }
84
85 BOOST_AUTO_TEST_CASE(obfuscation_serialize)
86 {
87 const Obfuscation original{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
88
89 // Serialization
90 DataStream ds;
91 ds << original;
92
93 BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::KEY_SIZE); // serialized as a vector
94
95 // Deserialization
96 Obfuscation recovered{};
97 ds >> recovered;
98
99 BOOST_CHECK_EQUAL(recovered.HexKey(), original.HexKey());
100 }
101
102 BOOST_AUTO_TEST_CASE(obfuscation_empty)
103 {
104 const Obfuscation null_obf{};
105 BOOST_CHECK(!null_obf);
106
107 const Obfuscation non_null_obf{"ff00ff00ff00ff00"_hex};
108 BOOST_CHECK(non_null_obf);
109 }
110
111 BOOST_AUTO_TEST_CASE(xor_file)
112 {
113 fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
114 auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
115 const std::vector<uint8_t> test1{1, 2, 3};
116 const std::vector<uint8_t> test2{4, 5};
117 const Obfuscation xor_pat{"ff00ff00ff00ff00"_hex};
118
119 {
120 // Check errors for missing file
121 AutoFile xor_file{raw_file("rb"), xor_pat};
122 BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullptr"});
123 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullptr"});
124 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullptr"});
125 }
126 {
127 #if 0
128 // Temporary workaround for https://github.com/limenka/limenka/issues/30210
129 const char* mode = "wb";
130 #else
131 const char* mode = "wbx";
132 #endif
133 AutoFile xor_file{raw_file(mode), xor_pat};
134 xor_file << test1 << test2;
135 BOOST_REQUIRE_EQUAL(xor_file.fclose(), 0);
136 }
137 {
138 // Read raw from disk
139 AutoFile non_xor_file{raw_file("rb")};
140 std::vector<std::byte> raw(7);
141 non_xor_file >> Span{raw};
142 BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa");
143 // Check that no padding exists
144 BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
145 }
146 {
147 AutoFile xor_file{raw_file("rb"), xor_pat};
148 std::vector<std::byte> read1, read2;
149 xor_file >> read1 >> read2;
150 BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
151 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2));
152 // Check that eof was reached
153 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
154 }
155 {
156 AutoFile xor_file{raw_file("rb"), xor_pat};
157 std::vector<std::byte> read2;
158 // Check that ignore works
159 xor_file.ignore(4);
160 xor_file >> read2;
161 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2));
162 // Check that ignore and read fail now
163 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
164 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
165 }
166 }
167
168 BOOST_AUTO_TEST_CASE(streams_vector_writer)
169 {
170 unsigned char a(1);
171 unsigned char b(2);
172 unsigned char bytes[] = { 3, 4, 5, 6 };
173 std::vector<unsigned char> vch;
174
175 // Each test runs twice. Serializing a second time at the same starting
176 // point should yield the same results, even if the first test grew the
177 // vector.
178
179 VectorWriter{vch, 0, a, b};
180 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}}));
181 VectorWriter{vch, 0, a, b};
182 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}}));
183 vch.clear();
184
185 VectorWriter{vch, 2, a, b};
186 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}}));
187 VectorWriter{vch, 2, a, b};
188 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}}));
189 vch.clear();
190
191 vch.resize(5, 0);
192 VectorWriter{vch, 2, a, b};
193 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}}));
194 VectorWriter{vch, 2, a, b};
195 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}}));
196 vch.clear();
197
198 vch.resize(4, 0);
199 VectorWriter{vch, 3, a, b};
200 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}}));
201 VectorWriter{vch, 3, a, b};
202 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}}));
203 vch.clear();
204
205 vch.resize(4, 0);
206 VectorWriter{vch, 4, a, b};
207 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}}));
208 VectorWriter{vch, 4, a, b};
209 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}}));
210 vch.clear();
211
212 VectorWriter{vch, 0, bytes};
213 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}}));
214 VectorWriter{vch, 0, bytes};
215 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}}));
216 vch.clear();
217
218 vch.resize(4, 8);
219 VectorWriter{vch, 2, a, bytes, b};
220 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}}));
221 VectorWriter{vch, 2, a, bytes, b};
222 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}}));
223 vch.clear();
224 }
225
226 BOOST_AUTO_TEST_CASE(streams_vector_reader)
227 {
228 std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6};
229
230 SpanReader reader{vch};
231 BOOST_CHECK_EQUAL(reader.size(), 6U);
232 BOOST_CHECK(!reader.empty());
233
234 // Read a single byte as an unsigned char.
235 unsigned char a;
236 reader >> a;
237 BOOST_CHECK_EQUAL(a, 1);
238 BOOST_CHECK_EQUAL(reader.size(), 5U);
239 BOOST_CHECK(!reader.empty());
240
241 // Read a single byte as a int8_t.
242 int8_t b;
243 reader >> b;
244 BOOST_CHECK_EQUAL(b, -1);
245 BOOST_CHECK_EQUAL(reader.size(), 4U);
246 BOOST_CHECK(!reader.empty());
247
248 // Read a 4 bytes as an unsigned int.
249 unsigned int c;
250 reader >> c;
251 BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256
252 BOOST_CHECK_EQUAL(reader.size(), 0U);
253 BOOST_CHECK(reader.empty());
254
255 // Reading after end of byte vector throws an error.
256 signed int d;
257 BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
258
259 // Read a 4 bytes as a signed int from the beginning of the buffer.
260 SpanReader new_reader{vch};
261 new_reader >> d;
262 BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256
263 BOOST_CHECK_EQUAL(new_reader.size(), 2U);
264 BOOST_CHECK(!new_reader.empty());
265
266 // Reading after end of byte vector throws an error even if the reader is
267 // not totally empty.
268 BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure);
269 }
270
271 BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue)
272 {
273 std::vector<uint8_t> data{0x82, 0xa7, 0x31};
274 SpanReader reader{data};
275 uint32_t varint = 0;
276 // Deserialize into r-value
277 reader >> VARINT(varint);
278 BOOST_CHECK_EQUAL(varint, 54321U);
279 BOOST_CHECK(reader.empty());
280 }
281
282 BOOST_AUTO_TEST_CASE(bitstream_reader_writer)
283 {
284 DataStream data{};
285
286 BitStreamWriter bit_writer{data};
287 bit_writer.Write(0, 1);
288 bit_writer.Write(2, 2);
289 bit_writer.Write(6, 3);
290 bit_writer.Write(11, 4);
291 bit_writer.Write(1, 5);
292 bit_writer.Write(32, 6);
293 bit_writer.Write(7, 7);
294 bit_writer.Write(30497, 16);
295 bit_writer.Flush();
296
297 DataStream data_copy{data};
298 uint32_t serialized_int1;
299 data >> serialized_int1;
300 BOOST_CHECK_EQUAL(serialized_int1, uint32_t{0x7700C35A}); // NOTE: Serialized as LE
301 uint16_t serialized_int2;
302 data >> serialized_int2;
303 BOOST_CHECK_EQUAL(serialized_int2, uint16_t{0x1072}); // NOTE: Serialized as LE
304
305 BitStreamReader bit_reader{data_copy};
306 BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U);
307 BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U);
308 BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U);
309 BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U);
310 BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U);
311 BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U);
312 BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U);
313 BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U);
314 BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure);
315 }
316
317 BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
318 {
319 std::vector<std::byte> in;
320
321 // Degenerate case
322 {
323 DataStream ds{in};
324 Obfuscation{}(ds);
325 BOOST_CHECK_EQUAL(""s, ds.str());
326 }
327
328 in.push_back(std::byte{0x0f});
329 in.push_back(std::byte{0xf0});
330
331 // Single character key
332 {
333 const Obfuscation obfuscation{"ffffffffffffffff"_hex};
334
335 DataStream ds{in};
336 obfuscation(ds);
337 BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
338 }
339
340 // Multi character key
341
342 in.clear();
343 in.push_back(std::byte{0xf0});
344 in.push_back(std::byte{0x0f});
345
346 {
347 const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex};
348
349 DataStream ds{in};
350 obfuscation(ds);
351 BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
352 }
353 }
354
355 BOOST_AUTO_TEST_CASE(streams_buffered_file)
356 {
357 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
358 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
359
360 // The value at each offset is the offset.
361 for (uint8_t j = 0; j < 40; ++j) {
362 file << j;
363 }
364 file.seek(0, SEEK_SET);
365
366 // The buffer size (second arg) must be greater than the rewind
367 // amount (third arg).
368 try {
369 BufferedFile bfbad{file, 25, 25};
370 BOOST_CHECK(false);
371 } catch (const std::exception& e) {
372 BOOST_CHECK(strstr(e.what(),
373 "Rewind limit must be less than buffer size") != nullptr);
374 }
375
376 // The buffer is 25 bytes, allow rewinding 10 bytes.
377 BufferedFile bf{file, 25, 10};
378 BOOST_CHECK(!bf.eof());
379
380 uint8_t i;
381 bf >> i;
382 BOOST_CHECK_EQUAL(i, 0);
383 bf >> i;
384 BOOST_CHECK_EQUAL(i, 1);
385
386 // After reading bytes 0 and 1, we're positioned at 2.
387 BOOST_CHECK_EQUAL(bf.GetPos(), 2U);
388
389 // Rewind to offset 0, ok (within the 10 byte window).
390 BOOST_CHECK(bf.SetPos(0));
391 bf >> i;
392 BOOST_CHECK_EQUAL(i, 0);
393
394 // We can go forward to where we've been, but beyond may fail.
395 BOOST_CHECK(bf.SetPos(2));
396 bf >> i;
397 BOOST_CHECK_EQUAL(i, 2);
398
399 // If you know the maximum number of bytes that should be
400 // read to deserialize the variable, you can limit the read
401 // extent. The current file offset is 3, so the following
402 // SetLimit() allows zero bytes to be read.
403 BOOST_CHECK(bf.SetLimit(3));
404 try {
405 bf >> i;
406 BOOST_CHECK(false);
407 } catch (const std::exception& e) {
408 BOOST_CHECK(strstr(e.what(),
409 "Attempt to position past buffer limit") != nullptr);
410 }
411 // The default argument removes the limit completely.
412 BOOST_CHECK(bf.SetLimit());
413 // The read position should still be at 3 (no change).
414 BOOST_CHECK_EQUAL(bf.GetPos(), 3U);
415
416 // Read from current offset, 3, forward until position 10.
417 for (uint8_t j = 3; j < 10; ++j) {
418 bf >> i;
419 BOOST_CHECK_EQUAL(i, j);
420 }
421 BOOST_CHECK_EQUAL(bf.GetPos(), 10U);
422
423 // We're guaranteed (just barely) to be able to rewind to zero.
424 BOOST_CHECK(bf.SetPos(0));
425 BOOST_CHECK_EQUAL(bf.GetPos(), 0U);
426 bf >> i;
427 BOOST_CHECK_EQUAL(i, 0);
428
429 // We can set the position forward again up to the farthest
430 // into the stream we've been, but no farther. (Attempting
431 // to go farther may succeed, but it's not guaranteed.)
432 BOOST_CHECK(bf.SetPos(10));
433 bf >> i;
434 BOOST_CHECK_EQUAL(i, 10);
435 BOOST_CHECK_EQUAL(bf.GetPos(), 11U);
436
437 // Now it's only guaranteed that we can rewind to offset 1
438 // (current read position, 11, minus rewind amount, 10).
439 BOOST_CHECK(bf.SetPos(1));
440 BOOST_CHECK_EQUAL(bf.GetPos(), 1U);
441 bf >> i;
442 BOOST_CHECK_EQUAL(i, 1);
443
444 // We can stream into large variables, even larger than
445 // the buffer size.
446 BOOST_CHECK(bf.SetPos(11));
447 {
448 uint8_t a[40 - 11];
449 bf >> a;
450 for (uint8_t j = 0; j < sizeof(a); ++j) {
451 BOOST_CHECK_EQUAL(a[j], 11 + j);
452 }
453 }
454 BOOST_CHECK_EQUAL(bf.GetPos(), 40U);
455
456 // We've read the entire file, the next read should throw.
457 try {
458 bf >> i;
459 BOOST_CHECK(false);
460 } catch (const std::exception& e) {
461 BOOST_CHECK(strstr(e.what(),
462 "BufferedFile::Fill: end of file") != nullptr);
463 }
464 // Attempting to read beyond the end sets the EOF indicator.
465 BOOST_CHECK(bf.eof());
466
467 // Still at offset 40, we can go back 10, to 30.
468 BOOST_CHECK_EQUAL(bf.GetPos(), 40U);
469 BOOST_CHECK(bf.SetPos(30));
470 bf >> i;
471 BOOST_CHECK_EQUAL(i, 30);
472 BOOST_CHECK_EQUAL(bf.GetPos(), 31U);
473
474 // We're too far to rewind to position zero.
475 BOOST_CHECK(!bf.SetPos(0));
476 // But we should now be positioned at least as far back as allowed
477 // by the rewind window (relative to our farthest read position, 40).
478 BOOST_CHECK(bf.GetPos() <= 30U);
479
480 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
481
482 fs::remove(streams_test_filename);
483 }
484
485 BOOST_AUTO_TEST_CASE(streams_buffered_file_skip)
486 {
487 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
488 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
489 // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01).
490 for (uint8_t j = 0; j < 40; ++j) {
491 file << j;
492 }
493 file.seek(0, SEEK_SET);
494
495 // The buffer is 25 bytes, allow rewinding 10 bytes.
496 BufferedFile bf{file, 25, 10};
497
498 uint8_t i;
499 // This is like bf >> (7-byte-variable), in that it will cause data
500 // to be read from the file into memory, but it's not copied to us.
501 bf.SkipTo(7);
502 BOOST_CHECK_EQUAL(bf.GetPos(), 7U);
503 bf >> i;
504 BOOST_CHECK_EQUAL(i, 7);
505
506 // The bytes in the buffer up to offset 7 are valid and can be read.
507 BOOST_CHECK(bf.SetPos(0));
508 bf >> i;
509 BOOST_CHECK_EQUAL(i, 0);
510 bf >> i;
511 BOOST_CHECK_EQUAL(i, 1);
512
513 bf.SkipTo(11);
514 bf >> i;
515 BOOST_CHECK_EQUAL(i, 11);
516
517 // SkipTo() honors the transfer limit; we can't position beyond the limit.
518 bf.SetLimit(13);
519 try {
520 bf.SkipTo(14);
521 BOOST_CHECK(false);
522 } catch (const std::exception& e) {
523 BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr);
524 }
525
526 // We can position exactly to the transfer limit.
527 bf.SkipTo(13);
528 BOOST_CHECK_EQUAL(bf.GetPos(), 13U);
529
530 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
531 fs::remove(streams_test_filename);
532 }
533
534 BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
535 {
536 // Make this test deterministic.
537 SeedRandomForTest(SeedRand::ZEROS);
538
539 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
540 for (int rep = 0; rep < 50; ++rep) {
541 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")};
542 size_t fileSize = m_rng.randrange(256);
543 for (uint8_t i = 0; i < fileSize; ++i) {
544 file << i;
545 }
546 file.seek(0, SEEK_SET);
547
548 size_t bufSize = m_rng.randrange(300) + 1;
549 size_t rewindSize = m_rng.randrange(bufSize);
550 BufferedFile bf{file, bufSize, rewindSize};
551 size_t currentPos = 0;
552 size_t maxPos = 0;
553 for (int step = 0; step < 100; ++step) {
554 if (currentPos >= fileSize)
555 break;
556
557 // We haven't read to the end of the file yet.
558 BOOST_CHECK(!bf.eof());
559 BOOST_CHECK_EQUAL(bf.GetPos(), currentPos);
560
561 // Pretend the file consists of a series of objects of varying
562 // sizes; the boundaries of the objects can interact arbitrarily
563 // with the CBufferFile's internal buffer. These first three
564 // cases simulate objects of various sizes (1, 2, 5 bytes).
565 switch (m_rng.randrange(6)) {
566 case 0: {
567 uint8_t a[1];
568 if (currentPos + 1 > fileSize)
569 continue;
570 bf.SetLimit(currentPos + 1);
571 bf >> a;
572 for (uint8_t i = 0; i < 1; ++i) {
573 BOOST_CHECK_EQUAL(a[i], currentPos);
574 currentPos++;
575 }
576 break;
577 }
578 case 1: {
579 uint8_t a[2];
580 if (currentPos + 2 > fileSize)
581 continue;
582 bf.SetLimit(currentPos + 2);
583 bf >> a;
584 for (uint8_t i = 0; i < 2; ++i) {
585 BOOST_CHECK_EQUAL(a[i], currentPos);
586 currentPos++;
587 }
588 break;
589 }
590 case 2: {
591 uint8_t a[5];
592 if (currentPos + 5 > fileSize)
593 continue;
594 bf.SetLimit(currentPos + 5);
595 bf >> a;
596 for (uint8_t i = 0; i < 5; ++i) {
597 BOOST_CHECK_EQUAL(a[i], currentPos);
598 currentPos++;
599 }
600 break;
601 }
602 case 3: {
603 // SkipTo is similar to the "read" cases above, except
604 // we don't receive the data.
605 size_t skip_length{static_cast<size_t>(m_rng.randrange(5))};
606 if (currentPos + skip_length > fileSize) continue;
607 bf.SetLimit(currentPos + skip_length);
608 bf.SkipTo(currentPos + skip_length);
609 currentPos += skip_length;
610 break;
611 }
612 case 4: {
613 // Find a byte value (that is at or ahead of the current position).
614 size_t find = currentPos + m_rng.randrange(8);
615 if (find >= fileSize)
616 find = fileSize - 1;
617 bf.FindByte(std::byte(find));
618 // The value at each offset is the offset.
619 BOOST_CHECK_EQUAL(bf.GetPos(), find);
620 currentPos = find;
621
622 bf.SetLimit(currentPos + 1);
623 uint8_t i;
624 bf >> i;
625 BOOST_CHECK_EQUAL(i, currentPos);
626 currentPos++;
627 break;
628 }
629 case 5: {
630 size_t requestPos = m_rng.randrange(maxPos + 4);
631 bool okay = bf.SetPos(requestPos);
632 // The new position may differ from the requested position
633 // because we may not be able to rewind beyond the rewind
634 // window, and we may not be able to move forward beyond the
635 // farthest position we've reached so far.
636 currentPos = bf.GetPos();
637 BOOST_CHECK_EQUAL(okay, currentPos == requestPos);
638 // Check that we can position within the rewind window.
639 if (requestPos <= maxPos &&
640 maxPos > rewindSize &&
641 requestPos >= maxPos - rewindSize) {
642 // We requested a position within the rewind window.
643 BOOST_CHECK(okay);
644 }
645 break;
646 }
647 }
648 if (maxPos < currentPos)
649 maxPos = currentPos;
650 }
651 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
652 }
653 fs::remove(streams_test_filename);
654 }
655
656 BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content)
657 {
658 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)};
659 const size_t buf_size{1 + m_rng.randrange(file_size)};
660 const FlatFilePos pos{0, 0};
661
662 const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE};
663 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
664
665 // Write out the file with random content
666 {
667 AutoFile f{test_file.Open(pos, /*read_only=*/false), obfuscation};
668 f.write(m_rng.randbytes<std::byte>(file_size));
669 BOOST_REQUIRE_EQUAL(f.fclose(), 0);
670 }
671 BOOST_CHECK_EQUAL(fs::file_size(test_file.FileName(pos)), file_size);
672
673 {
674 AutoFile direct_file{test_file.Open(pos, /*read_only=*/true), obfuscation};
675
676 AutoFile buffered_file{test_file.Open(pos, /*read_only=*/true), obfuscation};
677 BufferedReader buffered_reader{std::move(buffered_file), buf_size};
678
679 for (size_t total_read{0}; total_read < file_size;) {
680 const size_t read{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_read))};
681
682 DataBuffer direct_file_buffer{read};
683 direct_file.read(direct_file_buffer);
684
685 DataBuffer buffered_buffer{read};
686 buffered_reader.read(buffered_buffer);
687
688 BOOST_CHECK_EQUAL_COLLECTIONS(
689 direct_file_buffer.begin(), direct_file_buffer.end(),
690 buffered_buffer.begin(), buffered_buffer.end()
691 );
692
693 total_read += read;
694 }
695
696 {
697 DataBuffer excess_byte{1};
698 BOOST_CHECK_EXCEPTION(direct_file.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
699 }
700
701 {
702 DataBuffer excess_byte{1};
703 BOOST_CHECK_EXCEPTION(buffered_reader.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
704 }
705 }
706
707 fs::remove(test_file.FileName(pos));
708 }
709
710 BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content)
711 {
712 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)};
713 const size_t buf_size{1 + m_rng.randrange(file_size)};
714 const FlatFilePos pos{0, 0};
715
716 const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE};
717 const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE};
718 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()};
719
720 {
721 DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)};
722
723 AutoFile direct_file{test_direct.Open(pos, /*read_only=*/false), obfuscation};
724
725 AutoFile buffered_file{test_buffered.Open(pos, /*read_only=*/false), obfuscation};
726 {
727 BufferedWriter buffered{buffered_file, buf_size};
728
729 for (size_t total_written{0}; total_written < file_size;) {
730 const size_t write_size{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_written))};
731
732 auto current_span = std::span{test_data}.subspan(total_written, write_size);
733 direct_file.write(current_span);
734 buffered.write(current_span);
735
736 total_written += write_size;
737 }
738 }
739 BOOST_REQUIRE_EQUAL(buffered_file.fclose(), 0);
740 BOOST_REQUIRE_EQUAL(direct_file.fclose(), 0);
741 }
742
743 // Compare the resulting files
744 DataBuffer direct_result{file_size};
745 {
746 AutoFile verify_direct{test_direct.Open(pos, /*read_only=*/true), obfuscation};
747 verify_direct.read(direct_result);
748
749 DataBuffer excess_byte{1};
750 BOOST_CHECK_EXCEPTION(verify_direct.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
751 }
752
753 DataBuffer buffered_result{file_size};
754 {
755 AutoFile verify_buffered{test_buffered.Open(pos, /*read_only=*/true), obfuscation};
756 verify_buffered.read(buffered_result);
757
758 DataBuffer excess_byte{1};
759 BOOST_CHECK_EXCEPTION(verify_buffered.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
760 }
761
762 BOOST_CHECK_EQUAL_COLLECTIONS(
763 direct_result.begin(), direct_result.end(),
764 buffered_result.begin(), buffered_result.end()
765 );
766
767 fs::remove(test_direct.FileName(pos));
768 fs::remove(test_buffered.FileName(pos));
769 }
770
771 BOOST_AUTO_TEST_CASE(buffered_writer_reader)
772 {
773 const uint32_t v1{m_rng.rand32()}, v2{m_rng.rand32()}, v3{m_rng.rand32()};
774 const fs::path test_file{m_args.GetDataDirBase() / "test_buffered_write_read.bin"};
775
776 // Write out the values through a precisely sized BufferedWriter
777 AutoFile file{fsbridge::fopen(test_file, "w+b")};
778 {
779 BufferedWriter f(file, sizeof(v1) + sizeof(v2) + sizeof(v3));
780 f << v1 << v2;
781 f.write(std::as_bytes(std::span{&v3, 1}));
782 }
783 BOOST_REQUIRE_EQUAL(file.fclose(), 0);
784
785 // Read back and verify using BufferedReader
786 {
787 uint32_t _v1{0}, _v2{0}, _v3{0};
788 AutoFile file{fsbridge::fopen(test_file, "rb")};
789 BufferedReader f(std::move(file), sizeof(v1) + sizeof(v2) + sizeof(v3));
790 f >> _v1 >> _v2;
791 f.read(std::as_writable_bytes(std::span{&_v3, 1}));
792 BOOST_CHECK_EQUAL(_v1, v1);
793 BOOST_CHECK_EQUAL(_v2, v2);
794 BOOST_CHECK_EQUAL(_v3, v3);
795
796 DataBuffer excess_byte{1};
797 BOOST_CHECK_EXCEPTION(f.read(excess_byte), std::ios_base::failure, HasReason{"end of file"});
798 }
799
800 fs::remove(test_file);
801 }
802
803 BOOST_AUTO_TEST_CASE(streams_hashed)
804 {
805 DataStream stream{};
806 HashedSourceWriter hash_writer{stream};
807 const std::string data{"limenka"};
808 hash_writer << data;
809
810 HashVerifier hash_verifier{stream};
811 std::string result;
812 hash_verifier >> result;
813 BOOST_CHECK_EQUAL(data, result);
814 BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash());
815 }
816
817 BOOST_AUTO_TEST_SUITE_END()
818