serialize.h raw
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-present The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8
9 #include <attributes.h>
10 #include <compat/assumptions.h> // IWYU pragma: keep
11 #include <compat/endian.h>
12 #include <prevector.h>
13 #include <span.h>
14 #include <util/overflow.h>
15
16 #include <algorithm>
17 #include <concepts>
18 #include <cstdint>
19 #include <cstring>
20 #include <ios>
21 #include <limits>
22 #include <map>
23 #include <memory>
24 #include <set>
25 #include <span>
26 #include <string>
27 #include <string_view>
28 #include <utility>
29 #include <vector>
30
31 /**
32 * The maximum size of a serialized object in bytes or number of elements
33 * (for eg vectors) when the size is encoded as CompactSize.
34 */
35 static constexpr uint64_t MAX_SIZE = 0x02000000;
36
37 /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
38 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
39
40 /**
41 * Dummy data type to identify deserializing constructors.
42 *
43 * By convention, a constructor of a type T with signature
44 *
45 * template <typename Stream> T::T(deserialize_type, Stream& s)
46 *
47 * is a deserializing constructor, which builds the type by
48 * deserializing it from s. If T contains const fields, this
49 * is likely the only way to do so.
50 */
51 struct deserialize_type {};
52 constexpr deserialize_type deserialize {};
53
54 /*
55 * Lowest-level serialization and conversion.
56 */
57 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
58 {
59 s.write(std::as_bytes(std::span{&obj, 1}));
60 }
61 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
62 {
63 obj = htole16_internal(obj);
64 s.write(std::as_bytes(std::span{&obj, 1}));
65 }
66 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
67 {
68 obj = htole32_internal(obj);
69 s.write(std::as_bytes(std::span{&obj, 1}));
70 }
71 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
72 {
73 obj = htobe32_internal(obj);
74 s.write(std::as_bytes(std::span{&obj, 1}));
75 }
76 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
77 {
78 obj = htole64_internal(obj);
79 s.write(std::as_bytes(std::span{&obj, 1}));
80 }
81 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
82 {
83 uint8_t obj;
84 s.read(std::as_writable_bytes(std::span{&obj, 1}));
85 return obj;
86 }
87 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
88 {
89 uint16_t obj;
90 s.read(std::as_writable_bytes(std::span{&obj, 1}));
91 return le16toh_internal(obj);
92 }
93 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
94 {
95 uint32_t obj;
96 s.read(std::as_writable_bytes(std::span{&obj, 1}));
97 return le32toh_internal(obj);
98 }
99 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
100 {
101 uint32_t obj;
102 s.read(std::as_writable_bytes(std::span{&obj, 1}));
103 return be32toh_internal(obj);
104 }
105 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
106 {
107 uint64_t obj;
108 s.read(std::as_writable_bytes(std::span{&obj, 1}));
109 return le64toh_internal(obj);
110 }
111
112
113 class SizeComputer;
114
115 /**
116 * Convert any argument to a reference to X, maintaining constness.
117 *
118 * This can be used in serialization code to invoke a base class's
119 * serialization routines.
120 *
121 * Example use:
122 * class Base { ... };
123 * class Child : public Base {
124 * int m_data;
125 * public:
126 * SERIALIZE_METHODS(Child, obj) {
127 * READWRITE(AsBase<Base>(obj), obj.m_data);
128 * }
129 * };
130 *
131 * static_cast cannot easily be used here, as the type of Obj will be const Child&
132 * during serialization and Child& during deserialization. AsBase will convert to
133 * const Base& and Base& appropriately.
134 */
135 template <class Out, class In>
136 Out& AsBase(In& x)
137 {
138 static_assert(std::is_base_of_v<Out, In>);
139 return x;
140 }
141 template <class Out, class In>
142 const Out& AsBase(const In& x)
143 {
144 static_assert(std::is_base_of_v<Out, In>);
145 return x;
146 }
147
148 #define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__))
149 #define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, std::remove_const_t<Type>& obj) { code; })
150 #define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; })
151
152 /**
153 * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
154 *
155 * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
156 * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
157 * deserializing).
158 *
159 * Example use:
160 * struct FooFormatter {
161 * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
162 * }
163 * would define a class FooFormatter that defines a serialization of Class objects consisting
164 * of serializing its val1 member using the default serialization, and its val2 member using
165 * VARINT serialization. That FooFormatter can then be used in statements like
166 * READWRITE(Using<FooFormatter>(obj.bla)).
167 */
168 #define FORMATTER_METHODS(cls, obj) \
169 template<typename Stream> \
170 static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \
171 template<typename Stream> \
172 static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \
173 template<typename Stream, typename Type, typename Operation> \
174 static void SerializationOps(Type& obj, Stream& s, Operation ser_action)
175
176 /**
177 * Formatter methods can retrieve parameters attached to a stream using the
178 * SER_PARAMS(type) macro as long as the stream is created directly or
179 * indirectly with a parameter of that type. This permits making serialization
180 * depend on run-time context in a type-safe way.
181 *
182 * Example use:
183 * struct BarParameter { bool fancy; ... };
184 * struct Bar { ... };
185 * struct FooFormatter {
186 * FORMATTER_METHODS(Bar, obj) {
187 * auto& param = SER_PARAMS(BarParameter);
188 * if (param.fancy) {
189 * READWRITE(VARINT(obj.value));
190 * } else {
191 * READWRITE(obj.value);
192 * }
193 * }
194 * };
195 * which would then be invoked as
196 * READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo)))
197 *
198 * parameter(obj) can be invoked anywhere in the call stack; it is
199 * passed down recursively into all serialization code, until another
200 * serialization parameter overrides it.
201 *
202 * Parameters will be implicitly converted where appropriate. This means that
203 * "parent" serialization code can use a parameter that derives from, or is
204 * convertible to, a "child" formatter's parameter type.
205 *
206 * Compilation will fail in any context where serialization is invoked but
207 * no parameter of a type convertible to BarParameter is provided.
208 */
209 #define SER_PARAMS(type) (s.template GetParams<type>())
210
211 #define BASE_SERIALIZE_METHODS(cls) \
212 template <typename Stream> \
213 void Serialize(Stream& s) const \
214 { \
215 static_assert(std::is_same_v<const cls&, decltype(*this)>, "Serialize type mismatch"); \
216 Ser(s, *this); \
217 } \
218 template <typename Stream> \
219 void Unserialize(Stream& s) \
220 { \
221 static_assert(std::is_same_v<cls&, decltype(*this)>, "Unserialize type mismatch"); \
222 Unser(s, *this); \
223 }
224
225 /**
226 * Implement the Serialize and Unserialize methods by delegating to a single templated
227 * static method that takes the to-be-(de)serialized object as a parameter. This approach
228 * has the advantage that the constness of the object becomes a template parameter, and
229 * thus allows a single implementation that sees the object as const for serializing
230 * and non-const for deserializing, without casts.
231 */
232 #define SERIALIZE_METHODS(cls, obj) \
233 BASE_SERIALIZE_METHODS(cls) \
234 FORMATTER_METHODS(cls, obj)
235
236 // Templates for serializing to anything that looks like a stream,
237 // i.e. anything that supports .read(std::span<std::byte>) and .write(std::span<const std::byte>)
238 //
239
240 // Typically int8_t and char are distinct types, but some systems may define int8_t
241 // in terms of char. Forbid serialization of char in the typical case, but allow it if
242 // it's the only way to describe an int8_t.
243 template<class T>
244 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>;
245
246 // clang-format off
247 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
248 template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); }
249 template <typename Stream> void Serialize(Stream& s, int8_t a) { ser_writedata8(s, uint8_t(a)); }
250 template <typename Stream> void Serialize(Stream& s, uint8_t a) { ser_writedata8(s, a); }
251 template <typename Stream> void Serialize(Stream& s, int16_t a) { ser_writedata16(s, uint16_t(a)); }
252 template <typename Stream> void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
253 template <typename Stream> void Serialize(Stream& s, int32_t a) { ser_writedata32(s, uint32_t(a)); }
254 template <typename Stream> void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
255 template <typename Stream> void Serialize(Stream& s, int64_t a) { ser_writedata64(s, uint64_t(a)); }
256 template <typename Stream> void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
257
258 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); }
259 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); }
260 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, std::span<B, N> span) { s.write(std::as_bytes(span)); }
261 template <typename Stream, BasicByte B> void Serialize(Stream& s, std::span<B> span) { s.write(std::as_bytes(span)); }
262
263 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
264 template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte(ser_readdata8(s)); }
265 template <typename Stream> void Unserialize(Stream& s, int8_t& a) { a = int8_t(ser_readdata8(s)); }
266 template <typename Stream> void Unserialize(Stream& s, uint8_t& a) { a = ser_readdata8(s); }
267 template <typename Stream> void Unserialize(Stream& s, int16_t& a) { a = int16_t(ser_readdata16(s)); }
268 template <typename Stream> void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
269 template <typename Stream> void Unserialize(Stream& s, int32_t& a) { a = int32_t(ser_readdata32(s)); }
270 template <typename Stream> void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
271 template <typename Stream> void Unserialize(Stream& s, int64_t& a) { a = int64_t(ser_readdata64(s)); }
272 template <typename Stream> void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
273
274 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
275 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); }
276 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); }
277 template <typename Stream, BasicByte B> void Unserialize(Stream& s, std::span<B> span) { s.read(std::as_writable_bytes(span)); }
278
279 template <typename Stream> void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
280 template <typename Stream> void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
281 // clang-format on
282
283
284 /**
285 * Compact Size
286 * size < 253 -- 1 byte
287 * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes)
288 * size <= UINT_MAX -- 5 bytes (254 + 4 bytes)
289 * size > UINT_MAX -- 9 bytes (255 + 8 bytes)
290 */
291 constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
292 {
293 if (nSize < 253) return sizeof(unsigned char);
294 else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
295 else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
296 else return sizeof(unsigned char) + sizeof(uint64_t);
297 }
298
299 inline void WriteCompactSize(SizeComputer& os, uint64_t nSize);
300
301 template<typename Stream>
302 void WriteCompactSize(Stream& os, uint64_t nSize)
303 {
304 if (nSize < 253)
305 {
306 ser_writedata8(os, nSize);
307 }
308 else if (nSize <= std::numeric_limits<uint16_t>::max())
309 {
310 ser_writedata8(os, 253);
311 ser_writedata16(os, nSize);
312 }
313 else if (nSize <= std::numeric_limits<unsigned int>::max())
314 {
315 ser_writedata8(os, 254);
316 ser_writedata32(os, nSize);
317 }
318 else
319 {
320 ser_writedata8(os, 255);
321 ser_writedata64(os, nSize);
322 }
323 return;
324 }
325
326 /**
327 * Decode a CompactSize-encoded variable-length integer.
328 *
329 * As these are primarily used to encode the size of vector-like serializations, by default a range
330 * check is performed. When used as a generic number encoding, range_check should be set to false.
331 */
332 template<typename Stream>
333 uint64_t ReadCompactSize(Stream& is, bool range_check = true)
334 {
335 uint8_t chSize = ser_readdata8(is);
336 uint64_t nSizeRet = 0;
337 if (chSize < 253)
338 {
339 nSizeRet = chSize;
340 }
341 else if (chSize == 253)
342 {
343 nSizeRet = ser_readdata16(is);
344 if (nSizeRet < 253)
345 throw std::ios_base::failure("non-canonical ReadCompactSize()");
346 }
347 else if (chSize == 254)
348 {
349 nSizeRet = ser_readdata32(is);
350 if (nSizeRet < 0x10000u)
351 throw std::ios_base::failure("non-canonical ReadCompactSize()");
352 }
353 else
354 {
355 nSizeRet = ser_readdata64(is);
356 if (nSizeRet < 0x100000000ULL)
357 throw std::ios_base::failure("non-canonical ReadCompactSize()");
358 }
359 if (range_check && nSizeRet > MAX_SIZE) {
360 throw std::ios_base::failure("ReadCompactSize(): size too large");
361 }
362 return nSizeRet;
363 }
364
365 /**
366 * Variable-length integers: bytes are a MSB base-128 encoding of the number.
367 * The high bit in each byte signifies whether another digit follows. To make
368 * sure the encoding is one-to-one, one is subtracted from all but the last digit.
369 * Thus, the byte sequence a[] with length len, where all but the last byte
370 * has bit 128 set, encodes the number:
371 *
372 * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1))
373 *
374 * Properties:
375 * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes)
376 * * Every integer has exactly one encoding
377 * * Encoding does not depend on size of original integer type
378 * * No redundancy: every (infinite) byte sequence corresponds to a list
379 * of encoded integers.
380 *
381 * 0: [0x00] 256: [0x81 0x00]
382 * 1: [0x01] 16383: [0xFE 0x7F]
383 * 127: [0x7F] 16384: [0xFF 0x00]
384 * 128: [0x80 0x00] 16511: [0xFF 0x7F]
385 * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F]
386 * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00]
387 */
388
389 /**
390 * Mode for encoding VarInts.
391 *
392 * Currently there is no support for signed encodings. The default mode will not
393 * compile with signed values, and the legacy "nonnegative signed" mode will
394 * accept signed values, but improperly encode and decode them if they are
395 * negative. In the future, the DEFAULT mode could be extended to support
396 * negative numbers in a backwards compatible way, and additional modes could be
397 * added to support different varint formats (e.g. zigzag encoding).
398 */
399 enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED };
400
401 template <VarIntMode Mode, typename I>
402 struct CheckVarIntMode {
403 constexpr CheckVarIntMode()
404 {
405 static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned_v<I>, "Unsigned type required with mode DEFAULT.");
406 static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed_v<I>, "Signed type required with mode NONNEGATIVE_SIGNED.");
407 }
408 };
409
410 template<VarIntMode Mode, typename I>
411 inline unsigned int GetSizeOfVarInt(I n)
412 {
413 CheckVarIntMode<Mode, I>();
414 int nRet = 0;
415 while(true) {
416 nRet++;
417 if (n <= 0x7F)
418 break;
419 n = (n >> 7) - 1;
420 }
421 return nRet;
422 }
423
424 template<typename I>
425 inline void WriteVarInt(SizeComputer& os, I n);
426
427 template<typename Stream, VarIntMode Mode, typename I>
428 void WriteVarInt(Stream& os, I n)
429 {
430 CheckVarIntMode<Mode, I>();
431 unsigned char tmp[CeilDiv(sizeof(n) * 8, 7u)];
432 int len=0;
433 while(true) {
434 tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
435 if (n <= 0x7F)
436 break;
437 n = (n >> 7) - 1;
438 len++;
439 }
440 do {
441 ser_writedata8(os, tmp[len]);
442 } while(len--);
443 }
444
445 template<typename Stream, VarIntMode Mode, typename I>
446 I ReadVarInt(Stream& is)
447 {
448 CheckVarIntMode<Mode, I>();
449 I n = 0;
450 while(true) {
451 unsigned char chData = ser_readdata8(is);
452 if (n > (std::numeric_limits<I>::max() >> 7)) {
453 throw std::ios_base::failure("ReadVarInt(): size too large");
454 }
455 n = (n << 7) | (chData & 0x7F);
456 if (chData & 0x80) {
457 if (n == std::numeric_limits<I>::max()) {
458 throw std::ios_base::failure("ReadVarInt(): size too large");
459 }
460 n++;
461 } else {
462 return n;
463 }
464 }
465 }
466
467 /** Simple wrapper class to serialize objects using a formatter; used by Using(). */
468 template<typename Formatter, typename T>
469 class Wrapper
470 {
471 static_assert(std::is_lvalue_reference_v<T>, "Wrapper needs an lvalue reference type T");
472 protected:
473 T m_object;
474 public:
475 explicit Wrapper(T obj) : m_object(obj) {}
476 template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
477 template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
478 };
479
480 /** Cause serialization/deserialization of an object to be done using a specified formatter class.
481 *
482 * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for
483 * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
484 * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
485 *
486 * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
487 * const during serialization, and non-const during deserialization, which maintains const
488 * correctness.
489 */
490 template<typename Formatter, typename T>
491 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
492
493 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
494 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
495 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
496 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
497 #define LIMITED_VECTOR(obj,n) Using<LimitedVectorFormatter<n>>(obj)
498
499 /** Serialization wrapper class for integers in VarInt format. */
500 template<VarIntMode Mode>
501 struct VarIntFormatter
502 {
503 template<typename Stream, typename I> void Ser(Stream &s, I v)
504 {
505 WriteVarInt<Stream,Mode, std::remove_cv_t<I>>(s, v);
506 }
507
508 template<typename Stream, typename I> void Unser(Stream& s, I& v)
509 {
510 v = ReadVarInt<Stream,Mode, std::remove_cv_t<I>>(s);
511 }
512 };
513
514 /** Serialization wrapper class for custom integers and enums.
515 *
516 * It permits specifying the serialized size (1 to 8 bytes) and endianness.
517 *
518 * Use the big endian mode for values that are stored in memory in native
519 * byte order, but serialized in big endian notation. This is only intended
520 * to implement serializers that are compatible with existing formats, and
521 * its use is not recommended for new data structures.
522 */
523 template<int Bytes, bool BigEndian = false>
524 struct CustomUintFormatter
525 {
526 static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
527 static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
528
529 template <typename Stream, typename I> void Ser(Stream& s, I v)
530 {
531 if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
532 if (BigEndian) {
533 uint64_t raw = htobe64_internal(v);
534 s.write(std::as_bytes(std::span{&raw, 1}).last(Bytes));
535 } else {
536 uint64_t raw = htole64_internal(v);
537 s.write(std::as_bytes(std::span{&raw, 1}).first(Bytes));
538 }
539 }
540
541 template <typename Stream, typename I> void Unser(Stream& s, I& v)
542 {
543 using U = typename std::conditional_t<std::is_enum_v<I>, std::underlying_type<I>, std::common_type<I>>::type;
544 static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
545 uint64_t raw = 0;
546 if (BigEndian) {
547 s.read(std::as_writable_bytes(std::span{&raw, 1}).last(Bytes));
548 v = static_cast<I>(be64toh_internal(raw));
549 } else {
550 s.read(std::as_writable_bytes(std::span{&raw, 1}).first(Bytes));
551 v = static_cast<I>(le64toh_internal(raw));
552 }
553 }
554 };
555
556 template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>;
557
558 /** Formatter for integers in CompactSize format. */
559 template<bool RangeCheck>
560 struct CompactSizeFormatter
561 {
562 template<typename Stream, typename I>
563 void Unser(Stream& s, I& v)
564 {
565 uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
566 if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
567 throw std::ios_base::failure("CompactSize exceeds limit of type");
568 }
569 v = n;
570 }
571
572 template<typename Stream, typename I>
573 void Ser(Stream& s, I v)
574 {
575 static_assert(std::is_unsigned_v<I>, "CompactSize only supported for unsigned integers");
576 static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
577
578 WriteCompactSize<Stream>(s, v);
579 }
580 };
581
582 template <typename U, bool LOSSY = false>
583 struct ChronoFormatter {
584 template <typename Stream, typename Tp>
585 void Unser(Stream& s, Tp& tp)
586 {
587 U u;
588 s >> u;
589 // Lossy deserialization does not make sense, so force Wnarrowing
590 tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
591 }
592 template <typename Stream, typename Tp>
593 void Ser(Stream& s, Tp tp)
594 {
595 if constexpr (LOSSY) {
596 s << U(tp.time_since_epoch().count());
597 } else {
598 s << U{tp.time_since_epoch().count()};
599 }
600 }
601 };
602 template <typename U>
603 using LossyChronoFormatter = ChronoFormatter<U, true>;
604
605 class CompactSizeReader
606 {
607 protected:
608 uint64_t& n;
609 public:
610 explicit CompactSizeReader(uint64_t& n_in) : n(n_in) {}
611
612 template<typename Stream>
613 void Unserialize(Stream &s) const {
614 n = ReadCompactSize<Stream>(s);
615 }
616 };
617
618 class CompactSizeWriter
619 {
620 protected:
621 uint64_t n;
622 public:
623 explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
624
625 template<typename Stream>
626 void Serialize(Stream &s) const {
627 WriteCompactSize<Stream>(s, n);
628 }
629 };
630
631 template<size_t Limit>
632 struct LimitedStringFormatter
633 {
634 template<typename Stream>
635 void Unser(Stream& s, std::string& v)
636 {
637 size_t size = ReadCompactSize(s);
638 if (size > Limit) {
639 throw std::ios_base::failure("String length limit exceeded");
640 }
641 v.resize(size);
642 if (size != 0) s.read(MakeWritableByteSpan(v));
643 }
644
645 template<typename Stream>
646 void Ser(Stream& s, const std::string& v)
647 {
648 s << v;
649 }
650 };
651
652 /** Formatter to serialize/deserialize vector elements using another formatter
653 *
654 * Example:
655 * struct X {
656 * std::vector<uint64_t> v;
657 * SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
658 * };
659 * will define a struct that contains a vector of uint64_t, which is serialized
660 * as a vector of VarInt-encoded integers.
661 *
662 * V is not required to be an std::vector type. It works for any class that
663 * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
664 */
665 template<class Formatter>
666 struct VectorFormatter
667 {
668 template<typename Stream, typename V>
669 void Ser(Stream& s, const V& v)
670 {
671 Formatter formatter;
672 WriteCompactSize(s, v.size());
673 for (const typename V::value_type& elem : v) {
674 formatter.Ser(s, elem);
675 }
676 }
677
678 template<typename Stream, typename V>
679 void Unser(Stream& s, V& v)
680 {
681 Formatter formatter;
682 v.clear();
683 size_t size = ReadCompactSize(s);
684 size_t allocated = 0;
685 while (allocated < size) {
686 // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
687 // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
688 // X MiB of data to make us allocate X+5 Mib.
689 static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
690 allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
691 v.reserve(allocated);
692 while (v.size() < allocated) {
693 v.emplace_back();
694 formatter.Unser(s, v.back());
695 }
696 }
697 };
698 };
699
700 /**
701 * Forward declarations
702 */
703
704 /**
705 * string
706 */
707 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
708 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
709
710 /**
711 * string_view
712 */
713 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string_view<C>& str);
714 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string_view<C>& str) = delete;
715
716 /**
717 * prevector
718 */
719 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
720 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
721
722 /**
723 * vector
724 */
725 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
726 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
727
728 /**
729 * pair
730 */
731 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
732 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
733
734 /**
735 * map
736 */
737 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
738 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
739
740 /**
741 * set
742 */
743 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
744 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
745
746 /**
747 * shared_ptr
748 */
749 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
750 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
751
752 /**
753 * unique_ptr
754 */
755 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
756 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
757
758
759 /**
760 * If none of the specialized versions above matched, default to calling member function.
761 */
762 template <class T, class Stream>
763 concept Serializable = requires(T a, Stream s) { a.Serialize(s); };
764 template <typename Stream, typename T>
765 requires Serializable<T, Stream>
766 void Serialize(Stream& os, const T& a)
767 {
768 a.Serialize(os);
769 }
770
771 template <class T, class Stream>
772 concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); };
773 template <typename Stream, typename T>
774 requires Unserializable<T, Stream>
775 void Unserialize(Stream& is, T&& a)
776 {
777 a.Unserialize(is);
778 }
779
780 /** Default formatter. Serializes objects as themselves.
781 *
782 * The vector/prevector serialization code passes this to VectorFormatter
783 * to enable reusing that logic. It shouldn't be needed elsewhere.
784 */
785 struct DefaultFormatter
786 {
787 template<typename Stream, typename T>
788 static void Ser(Stream& s, const T& t) { Serialize(s, t); }
789
790 template<typename Stream, typename T>
791 static void Unser(Stream& s, T& t) { Unserialize(s, t); }
792 };
793
794 /**
795 * Limited vector formatter. Throws an error if a vector is oversized.
796 */
797
798 template<size_t Limit, class Formatter = DefaultFormatter>
799 struct LimitedVectorFormatter
800 {
801 template<typename Stream, typename V>
802 void Unser(Stream& s, V& v)
803 {
804 Formatter formatter;
805 v.clear();
806 size_t size = ReadCompactSize(s);
807 if (size > Limit) {
808 throw std::ios_base::failure("Vector length limit exceeded");
809 }
810 v.reserve(size);
811 for (size_t i = 0; i < size; ++i) {
812 v.emplace_back();
813 formatter.Unser(s, v.back());
814 }
815 }
816
817 template<typename Stream, typename V>
818 void Ser(Stream& s, const V& v)
819 {
820 VectorFormatter<Formatter>{}.Ser(s, v);
821 }
822 };
823
824
825
826
827 /**
828 * string
829 */
830 template<typename Stream, typename C>
831 void Serialize(Stream& os, const std::basic_string<C>& str)
832 {
833 WriteCompactSize(os, str.size());
834 if (!str.empty())
835 os.write(MakeByteSpan(str));
836 }
837
838 template<typename Stream, typename C>
839 void Unserialize(Stream& is, std::basic_string<C>& str)
840 {
841 unsigned int nSize = ReadCompactSize(is);
842 str.resize(nSize);
843 if (nSize != 0)
844 is.read(MakeWritableByteSpan(str));
845 }
846
847 /**
848 * string_view
849 */
850 template<typename Stream, typename C>
851 void Serialize(Stream& os, const std::basic_string_view<C>& str)
852 {
853 WriteCompactSize(os, str.size());
854 if (!str.empty()) {
855 os.write(MakeByteSpan(str));
856 }
857 }
858
859 /**
860 * prevector
861 */
862 template <typename Stream, unsigned int N, typename T>
863 void Serialize(Stream& os, const prevector<N, T>& v)
864 {
865 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
866 WriteCompactSize(os, v.size());
867 if (!v.empty()) os.write(MakeByteSpan(v));
868 } else {
869 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
870 }
871 }
872
873
874 template <typename Stream, unsigned int N, typename T>
875 void Unserialize(Stream& is, prevector<N, T>& v)
876 {
877 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
878 // Limit size per read so bogus size value won't cause out of memory
879 v.clear();
880 unsigned int nSize = ReadCompactSize(is);
881 unsigned int i = 0;
882 while (i < nSize) {
883 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
884 v.resize_uninitialized(i + blk);
885 is.read(std::as_writable_bytes(std::span{&v[i], blk}));
886 i += blk;
887 }
888 } else {
889 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
890 }
891 }
892
893
894 /**
895 * vector
896 */
897 template <typename Stream, typename T, typename A>
898 void Serialize(Stream& os, const std::vector<T, A>& v)
899 {
900 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
901 WriteCompactSize(os, v.size());
902 if (!v.empty()) os.write(MakeByteSpan(v));
903 } else if constexpr (std::is_same_v<T, bool>) {
904 // A special case for std::vector<bool>, as dereferencing
905 // std::vector<bool>::const_iterator does not result in a const bool&
906 // due to std::vector's special casing for bool arguments.
907 WriteCompactSize(os, v.size());
908 for (bool elem : v) {
909 ::Serialize(os, elem);
910 }
911 } else {
912 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
913 }
914 }
915
916
917 template <typename Stream, typename T, typename A>
918 void Unserialize(Stream& is, std::vector<T, A>& v)
919 {
920 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
921 // Limit size per read so bogus size value won't cause out of memory
922 v.clear();
923 unsigned int nSize = ReadCompactSize(is);
924 unsigned int i = 0;
925 while (i < nSize) {
926 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
927 v.resize(i + blk);
928 is.read(std::as_writable_bytes(std::span{&v[i], blk}));
929 i += blk;
930 }
931 } else {
932 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
933 }
934 }
935
936
937 /**
938 * pair
939 */
940 template<typename Stream, typename K, typename T>
941 void Serialize(Stream& os, const std::pair<K, T>& item)
942 {
943 Serialize(os, item.first);
944 Serialize(os, item.second);
945 }
946
947 template<typename Stream, typename K, typename T>
948 void Unserialize(Stream& is, std::pair<K, T>& item)
949 {
950 Unserialize(is, item.first);
951 Unserialize(is, item.second);
952 }
953
954
955
956 /**
957 * map
958 */
959 template<typename Stream, typename K, typename T, typename Pred, typename A>
960 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
961 {
962 WriteCompactSize(os, m.size());
963 for (const auto& entry : m)
964 Serialize(os, entry);
965 }
966
967 template<typename Stream, typename K, typename T, typename Pred, typename A>
968 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
969 {
970 m.clear();
971 unsigned int nSize = ReadCompactSize(is);
972 typename std::map<K, T, Pred, A>::iterator mi = m.begin();
973 for (unsigned int i = 0; i < nSize; i++)
974 {
975 std::pair<K, T> item;
976 Unserialize(is, item);
977 mi = m.insert(mi, item);
978 }
979 }
980
981
982
983 /**
984 * set
985 */
986 template<typename Stream, typename K, typename Pred, typename A>
987 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
988 {
989 WriteCompactSize(os, m.size());
990 for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
991 Serialize(os, (*it));
992 }
993
994 template<typename Stream, typename K, typename Pred, typename A>
995 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
996 {
997 m.clear();
998 unsigned int nSize = ReadCompactSize(is);
999 typename std::set<K, Pred, A>::iterator it = m.begin();
1000 for (unsigned int i = 0; i < nSize; i++)
1001 {
1002 K key;
1003 Unserialize(is, key);
1004 it = m.insert(it, key);
1005 }
1006 }
1007
1008
1009
1010 /**
1011 * unique_ptr
1012 */
1013 template<typename Stream, typename T> void
1014 Serialize(Stream& os, const std::unique_ptr<const T>& p)
1015 {
1016 Serialize(os, *p);
1017 }
1018
1019 template<typename Stream, typename T>
1020 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
1021 {
1022 p.reset(new T(deserialize, is));
1023 }
1024
1025
1026
1027 /**
1028 * shared_ptr
1029 */
1030 template<typename Stream, typename T> void
1031 Serialize(Stream& os, const std::shared_ptr<const T>& p)
1032 {
1033 Serialize(os, *p);
1034 }
1035
1036 template<typename Stream, typename T>
1037 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
1038 {
1039 p = std::make_shared<const T>(deserialize, is);
1040 }
1041
1042 /**
1043 * Support for (un)serializing many things at once
1044 */
1045
1046 template <typename Stream, typename... Args>
1047 void SerializeMany(Stream& s, const Args&... args)
1048 {
1049 (::Serialize(s, args), ...);
1050 }
1051
1052 template <typename Stream, typename... Args>
1053 inline void UnserializeMany(Stream& s, Args&&... args)
1054 {
1055 (::Unserialize(s, args), ...);
1056 }
1057
1058 /**
1059 * Support for all macros providing or using the ser_action parameter of the SerializationOps method.
1060 */
1061 struct ActionSerialize {
1062 static constexpr bool ForRead() { return false; }
1063
1064 template<typename Stream, typename... Args>
1065 static void SerReadWriteMany(Stream& s, const Args&... args)
1066 {
1067 ::SerializeMany(s, args...);
1068 }
1069
1070 template<typename Stream, typename Type, typename Fn>
1071 static void SerRead(Stream& s, Type&&, Fn&&)
1072 {
1073 }
1074
1075 template<typename Stream, typename Type, typename Fn>
1076 static void SerWrite(Stream& s, Type&& obj, Fn&& fn)
1077 {
1078 fn(s, std::forward<Type>(obj));
1079 }
1080 };
1081 struct ActionUnserialize {
1082 static constexpr bool ForRead() { return true; }
1083
1084 template<typename Stream, typename... Args>
1085 static void SerReadWriteMany(Stream& s, Args&&... args)
1086 {
1087 ::UnserializeMany(s, args...);
1088 }
1089
1090 template<typename Stream, typename Type, typename Fn>
1091 static void SerRead(Stream& s, Type&& obj, Fn&& fn)
1092 {
1093 fn(s, std::forward<Type>(obj));
1094 }
1095
1096 template<typename Stream, typename Type, typename Fn>
1097 static void SerWrite(Stream& s, Type&&, Fn&&)
1098 {
1099 }
1100 };
1101
1102 /* ::GetSerializeSize implementations
1103 *
1104 * Computing the serialized size of objects is done through a special stream
1105 * object of type SizeComputer, which only records the number of bytes written
1106 * to it.
1107 *
1108 * If your Serialize or SerializationOp method has non-trivial overhead for
1109 * serialization, it may be worthwhile to implement a specialized version for
1110 * SizeComputer, which uses the s.seek() method to record bytes that would
1111 * be written instead.
1112 */
1113 class SizeComputer
1114 {
1115 protected:
1116 uint64_t m_size{0};
1117
1118 public:
1119 SizeComputer() = default;
1120
1121 void write(std::span<const std::byte> src)
1122 {
1123 m_size += src.size();
1124 }
1125
1126 /** Pretend this many bytes are written, without specifying them. */
1127 void seek(uint64_t num)
1128 {
1129 m_size += num;
1130 }
1131
1132 template <typename T>
1133 SizeComputer& operator<<(const T& obj)
1134 {
1135 ::Serialize(*this, obj);
1136 return *this;
1137 }
1138
1139 uint64_t size() const
1140 {
1141 return m_size;
1142 }
1143 };
1144
1145 template<typename I>
1146 inline void WriteVarInt(SizeComputer &s, I n)
1147 {
1148 s.seek(GetSizeOfVarInt<I>(n));
1149 }
1150
1151 inline void WriteCompactSize(SizeComputer &s, uint64_t nSize)
1152 {
1153 s.seek(GetSizeOfCompactSize(nSize));
1154 }
1155
1156 template <typename T>
1157 uint64_t GetSerializeSize(const T& t)
1158 {
1159 return (SizeComputer() << t).size();
1160 }
1161
1162 //! Check if type contains a stream by seeing if has a GetStream() method.
1163 template<typename T>
1164 concept ContainsStream = requires(T t) { t.GetStream(); };
1165
1166 /** Wrapper that overrides the GetParams() function of a stream. */
1167 template <typename SubStream, typename Params>
1168 class ParamsStream
1169 {
1170 const Params& m_params;
1171 // If ParamsStream constructor is passed an lvalue argument, Substream will
1172 // be a reference type, and m_substream will reference that argument.
1173 // Otherwise m_substream will be a substream instance and move from the
1174 // argument. Letting ParamsStream contain a substream instance instead of
1175 // just a reference is useful to make the ParamsStream object self contained
1176 // and let it do cleanup when destroyed, for example by closing files if
1177 // SubStream is a file stream.
1178 SubStream m_substream;
1179
1180 public:
1181 ParamsStream(SubStream&& substream, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{std::forward<SubStream>(substream)} {}
1182
1183 template <typename NestedSubstream, typename Params1, typename Params2, typename... NestedParams>
1184 ParamsStream(NestedSubstream&& s, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const NestedParams&... params LIFETIMEBOUND)
1185 : ParamsStream{::ParamsStream{std::forward<NestedSubstream>(s), params2, params...}, params1} {}
1186
1187 template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; }
1188 template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; }
1189 void write(std::span<const std::byte> src) { GetStream().write(src); }
1190 void read(std::span<std::byte> dst) { GetStream().read(dst); }
1191 void ignore(size_t num) { GetStream().ignore(num); }
1192 bool empty() const { return GetStream().empty(); }
1193 size_t size() const { return GetStream().size(); }
1194
1195 //! Get reference to stream parameters.
1196 template <typename P>
1197 const auto& GetParams() const
1198 {
1199 if constexpr (std::is_convertible_v<Params, P>) {
1200 return m_params;
1201 } else {
1202 return m_substream.template GetParams<P>();
1203 }
1204 }
1205
1206 //! Get reference to underlying stream.
1207 auto& GetStream()
1208 {
1209 if constexpr (ContainsStream<SubStream>) {
1210 return m_substream.GetStream();
1211 } else {
1212 return m_substream;
1213 }
1214 }
1215 const auto& GetStream() const
1216 {
1217 if constexpr (ContainsStream<SubStream>) {
1218 return m_substream.GetStream();
1219 } else {
1220 return m_substream;
1221 }
1222 }
1223 };
1224
1225 /**
1226 * Explicit template deduction guide is required for single-parameter
1227 * constructor so Substream&& is treated as a forwarding reference, and
1228 * SubStream is deduced as reference type for lvalue arguments.
1229 */
1230 template <typename Substream, typename Params>
1231 ParamsStream(Substream&&, const Params&) -> ParamsStream<Substream, Params>;
1232
1233 /**
1234 * Template deduction guide for multiple params arguments that creates a nested
1235 * ParamsStream.
1236 */
1237 template <typename Substream, typename Params1, typename Params2, typename... Params>
1238 ParamsStream(Substream&& s, const Params1& params1, const Params2& params2, const Params&... params) ->
1239 ParamsStream<decltype(ParamsStream{std::forward<Substream>(s), params2, params...}), Params1>;
1240
1241 /** Wrapper that serializes objects with the specified parameters. */
1242 template <typename Params, typename T>
1243 class ParamsWrapper
1244 {
1245 const Params& m_params;
1246 T& m_object;
1247
1248 public:
1249 explicit ParamsWrapper(const Params& params, T& obj) : m_params{params}, m_object{obj} {}
1250
1251 template <typename Stream>
1252 void Serialize(Stream& s) const
1253 {
1254 ParamsStream ss{s, m_params};
1255 ::Serialize(ss, m_object);
1256 }
1257 template <typename Stream>
1258 void Unserialize(Stream& s)
1259 {
1260 ParamsStream ss{s, m_params};
1261 ::Unserialize(ss, m_object);
1262 }
1263 };
1264
1265 /**
1266 * Helper macro for SerParams structs
1267 *
1268 * Allows you define SerParams instances and then apply them directly
1269 * to an object via function call syntax, eg:
1270 *
1271 * constexpr SerParams FOO{....};
1272 * ss << FOO(obj);
1273 */
1274 #define SER_PARAMS_OPFUNC \
1275 /** \
1276 * Return a wrapper around t that (de)serializes it with specified parameter params. \
1277 * \
1278 * See SER_PARAMS for more information on serialization parameters. \
1279 */ \
1280 template <typename T> \
1281 auto operator()(T&& t) const \
1282 { \
1283 return ParamsWrapper{*this, t}; \
1284 }
1285
1286 #endif // BITCOIN_SERIALIZE_H
1287