options.go raw
1 /*
2 * SPDX-FileCopyrightText: © Hypermode Inc. <hello@hypermode.com>
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6 package badger
7
8 import (
9 "fmt"
10 "os"
11 "reflect"
12 "strconv"
13 "strings"
14 "time"
15
16 "github.com/dgraph-io/badger/v4/options"
17 "github.com/dgraph-io/badger/v4/table"
18 "github.com/dgraph-io/badger/v4/y"
19 "github.com/dgraph-io/ristretto/v2/z"
20 )
21
22 // Note: If you add a new option X make sure you also add a WithX method on Options.
23
24 // Options are params for creating DB object.
25 //
26 // This package provides DefaultOptions which contains options that should
27 // work for most applications. Consider using that as a starting point before
28 // customizing it for your own needs.
29 //
30 // Each option X is documented on the WithX method.
31 type Options struct {
32 testOnlyOptions
33
34 // Required options.
35
36 Dir string
37 ValueDir string
38
39 // Usually modified options.
40
41 SyncWrites bool
42 NumVersionsToKeep int
43 ReadOnly bool
44 Logger Logger
45 Compression options.CompressionType
46 InMemory bool
47 MetricsEnabled bool
48 // Sets the Stream.numGo field
49 NumGoroutines int
50
51 // Fine tuning options.
52
53 MemTableSize int64
54 BaseTableSize int64
55 BaseLevelSize int64
56 LevelSizeMultiplier int
57 TableSizeMultiplier int
58 MaxLevels int
59
60 VLogPercentile float64
61 ValueThreshold int64
62 NumMemtables int
63 // Changing BlockSize across DB runs will not break badger. The block size is
64 // read from the block index stored at the end of the table.
65 BlockSize int
66 BloomFalsePositive float64
67 BlockCacheSize int64
68 IndexCacheSize int64
69
70 NumLevelZeroTables int
71 NumLevelZeroTablesStall int
72
73 ValueLogFileSize int64
74 ValueLogMaxEntries uint32
75
76 NumCompactors int
77 CompactL0OnClose bool
78 LmaxCompaction bool
79 ZSTDCompressionLevel int
80
81 // When set, checksum will be validated for each entry read from the value log file.
82 VerifyValueChecksum bool
83
84 // Encryption related options.
85 EncryptionKey []byte // encryption key
86 EncryptionKeyRotationDuration time.Duration // key rotation duration
87
88 // BypassLockGuard will bypass the lock guard on badger. Bypassing lock
89 // guard can cause data corruption if multiple badger instances are using
90 // the same directory. Use this options with caution.
91 BypassLockGuard bool
92
93 // ChecksumVerificationMode decides when db should verify checksums for SSTable blocks.
94 ChecksumVerificationMode options.ChecksumVerificationMode
95
96 // DetectConflicts determines whether the transactions would be checked for
97 // conflicts. The transactions can be processed at a higher rate when
98 // conflict detection is disabled.
99 DetectConflicts bool
100
101 // NamespaceOffset specifies the offset from where the next 8 bytes contains the namespace.
102 NamespaceOffset int
103
104 // Magic version used by the application using badger to ensure that it doesn't open the DB
105 // with incompatible data format.
106 ExternalMagicVersion uint16
107
108 // Transaction start and commit timestamps are managed by end-user.
109 // This is only useful for databases built on top of Badger (like Dgraph).
110 // Not recommended for most users.
111 managedTxns bool
112
113 // 4. Flags for testing purposes
114 // ------------------------------
115 maxBatchCount int64 // max entries in batch
116 maxBatchSize int64 // max batch size in bytes
117
118 maxValueThreshold float64
119 }
120
121 // DefaultOptions sets a list of recommended options for good performance.
122 // Feel free to modify these to suit your needs with the WithX methods.
123 func DefaultOptions(path string) Options {
124 return Options{
125 Dir: path,
126 ValueDir: path,
127
128 MemTableSize: 64 << 20,
129 BaseTableSize: 2 << 20,
130 BaseLevelSize: 10 << 20,
131 TableSizeMultiplier: 2,
132 LevelSizeMultiplier: 10,
133 MaxLevels: 7,
134 NumGoroutines: 8,
135 MetricsEnabled: true,
136
137 NumCompactors: 4, // Run at least 2 compactors. Zero-th compactor prioritizes L0.
138 NumLevelZeroTables: 5,
139 NumLevelZeroTablesStall: 15,
140 NumMemtables: 5,
141 BloomFalsePositive: 0.01,
142 BlockSize: 4 * 1024,
143 SyncWrites: false,
144 NumVersionsToKeep: 1,
145 CompactL0OnClose: false,
146 VerifyValueChecksum: false,
147 Compression: options.Snappy,
148 BlockCacheSize: 256 << 20,
149 IndexCacheSize: 0,
150
151 // The following benchmarks were done on a 4 KB block size (default block size). The
152 // compression is ratio supposed to increase with increasing compression level but since the
153 // input for compression algorithm is small (4 KB), we don't get significant benefit at
154 // level 3.
155 // NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid.
156 // no_compression-16 10 502848865 ns/op 165.46 MB/s -
157 // zstd_compression/level_1-16 7 739037966 ns/op 112.58 MB/s 2.93
158 // zstd_compression/level_3-16 7 756950250 ns/op 109.91 MB/s 2.72
159 // zstd_compression/level_15-16 1 11135686219 ns/op 7.47 MB/s 4.38
160 // Benchmark code can be found in table/builder_test.go file
161 ZSTDCompressionLevel: 1,
162
163 // (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32.
164 // -1 so 2*ValueLogFileSize won't overflow on 32-bit systems.
165 ValueLogFileSize: 1<<30 - 1,
166
167 ValueLogMaxEntries: 1000000,
168
169 VLogPercentile: 0.0,
170 ValueThreshold: maxValueThreshold,
171
172 Logger: defaultLogger(INFO),
173 EncryptionKey: []byte{},
174 EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
175 DetectConflicts: true,
176 NamespaceOffset: -1,
177 }
178 }
179
180 func buildTableOptions(db *DB) table.Options {
181 opt := db.opt
182 dk, err := db.registry.LatestDataKey()
183 y.Check(err)
184 return table.Options{
185 ReadOnly: opt.ReadOnly,
186 MetricsEnabled: db.opt.MetricsEnabled,
187 TableSize: uint64(opt.BaseTableSize),
188 BlockSize: opt.BlockSize,
189 BloomFalsePositive: opt.BloomFalsePositive,
190 ChkMode: opt.ChecksumVerificationMode,
191 Compression: opt.Compression,
192 ZSTDCompressionLevel: opt.ZSTDCompressionLevel,
193 BlockCache: db.blockCache,
194 IndexCache: db.indexCache,
195 AllocPool: db.allocPool,
196 DataKey: dk,
197 }
198 }
199
200 const (
201 maxValueThreshold = (1 << 20) // 1 MB
202 )
203
204 // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold
205 // so values would be collocated with the LSM tree, with value log largely acting
206 // as a write-ahead log only. These options would reduce the disk usage of value
207 // log, and make Badger act more like a typical LSM tree.
208 func LSMOnlyOptions(path string) Options {
209 // Let's not set any other options, because they can cause issues with the
210 // size of key-value a user can pass to Badger. For e.g., if we set
211 // ValueLogFileSize to 64MB, a user can't pass a value more than that.
212 // Setting it to ValueLogMaxEntries to 1000, can generate too many files.
213 // These options are better configured on a usage basis, than broadly here.
214 // The ValueThreshold is the most important setting a user needs to do to
215 // achieve a heavier usage of LSM tree.
216 // NOTE: If a user does not want to set 64KB as the ValueThreshold because
217 // of performance reasons, 1KB would be a good option too, allowing
218 // values smaller than 1KB to be collocated with the keys in the LSM tree.
219 return DefaultOptions(path).WithValueThreshold(maxValueThreshold /* 1 MB */)
220 }
221
222 // parseCompression returns badger.compressionType and compression level given compression string
223 // of format compression-type:compression-level
224 func parseCompression(cStr string) (options.CompressionType, int, error) {
225 cStrSplit := strings.Split(cStr, ":")
226 cType := cStrSplit[0]
227 level := 3
228
229 var err error
230 if len(cStrSplit) == 2 {
231 level, err = strconv.Atoi(cStrSplit[1])
232 y.Check(err)
233 if level <= 0 {
234 return 0, 0,
235 fmt.Errorf("ERROR: compression level(%v) must be greater than zero", level)
236 }
237 } else if len(cStrSplit) > 2 {
238 return 0, 0, fmt.Errorf("ERROR: Invalid badger.compression argument")
239 }
240 switch cType {
241 case "zstd":
242 return options.ZSTD, level, nil
243 case "snappy":
244 return options.Snappy, 0, nil
245 case "none":
246 return options.None, 0, nil
247 }
248 return 0, 0, fmt.Errorf("ERROR: compression type (%s) invalid", cType)
249 }
250
251 // generateSuperFlag generates an identical SuperFlag string from the provided Options.
252 func generateSuperFlag(options Options) string {
253 superflag := ""
254 v := reflect.ValueOf(&options).Elem()
255 optionsStruct := v.Type()
256 for i := 0; i < v.NumField(); i++ {
257 if field := v.Field(i); field.CanInterface() {
258 name := strings.ToLower(optionsStruct.Field(i).Name)
259 kind := v.Field(i).Kind()
260 switch kind {
261 case reflect.Bool:
262 superflag += name + "="
263 superflag += fmt.Sprintf("%v; ", field.Bool())
264 case reflect.Int, reflect.Int64:
265 superflag += name + "="
266 superflag += fmt.Sprintf("%v; ", field.Int())
267 case reflect.Uint32, reflect.Uint64:
268 superflag += name + "="
269 superflag += fmt.Sprintf("%v; ", field.Uint())
270 case reflect.Float64:
271 superflag += name + "="
272 superflag += fmt.Sprintf("%v; ", field.Float())
273 case reflect.String:
274 superflag += name + "="
275 superflag += fmt.Sprintf("%v; ", field.String())
276 default:
277 continue
278 }
279 }
280 }
281 return superflag
282 }
283
284 // FromSuperFlag fills Options fields for each flag within the superflag. For
285 // example, replacing the default Options.NumGoroutines:
286 //
287 // options := FromSuperFlag("numgoroutines=4", DefaultOptions(""))
288 //
289 // It's important to note that if you pass an empty Options struct, FromSuperFlag
290 // will not fill it with default values. FromSuperFlag only writes to the fields
291 // present within the superflag string (case insensitive).
292 //
293 // It specially handles compression subflag.
294 // Valid options are {none,snappy,zstd:<level>}
295 // Example: compression=zstd:3;
296 // Unsupported: Options.Logger, Options.EncryptionKey
297 func (opt Options) FromSuperFlag(superflag string) Options {
298 // currentOptions act as a default value for the options superflag.
299 currentOptions := generateSuperFlag(opt)
300 currentOptions += "compression=;"
301
302 flags := z.NewSuperFlag(superflag).MergeAndCheckDefault(currentOptions)
303 v := reflect.ValueOf(&opt).Elem()
304 optionsStruct := v.Type()
305 for i := 0; i < v.NumField(); i++ {
306 // only iterate over exported fields
307 if field := v.Field(i); field.CanInterface() {
308 // z.SuperFlag stores keys as lowercase, keep everything case
309 // insensitive
310 name := strings.ToLower(optionsStruct.Field(i).Name)
311 if name == "compression" {
312 // We will specially handle this later. Skip it here.
313 continue
314 }
315 kind := v.Field(i).Kind()
316 switch kind {
317 case reflect.Bool:
318 field.SetBool(flags.GetBool(name))
319 case reflect.Int, reflect.Int64:
320 field.SetInt(flags.GetInt64(name))
321 case reflect.Uint32, reflect.Uint64:
322 field.SetUint(flags.GetUint64(name))
323 case reflect.Float64:
324 field.SetFloat(flags.GetFloat64(name))
325 case reflect.String:
326 field.SetString(flags.GetString(name))
327 }
328 }
329 }
330
331 // Only update the options for special flags that were present in the input superflag.
332 inputFlag := z.NewSuperFlag(superflag)
333 if inputFlag.Has("compression") {
334 ctype, clevel, err := parseCompression(flags.GetString("compression"))
335 switch err {
336 case nil:
337 opt.Compression = ctype
338 opt.ZSTDCompressionLevel = clevel
339 default:
340 ctype = options.CompressionType(flags.GetUint32("compression"))
341 y.AssertTruef(ctype <= 2, "ERROR: Invalid format or compression type. Got: %s",
342 flags.GetString("compression"))
343 opt.Compression = ctype
344 }
345 }
346
347 return opt
348 }
349
350 // WithDir returns a new Options value with Dir set to the given value.
351 //
352 // Dir is the path of the directory where key data will be stored in.
353 // If it doesn't exist, Badger will try to create it for you.
354 // This is set automatically to be the path given to `DefaultOptions`.
355 func (opt Options) WithDir(val string) Options {
356 opt.Dir = val
357 return opt
358 }
359
360 // WithValueDir returns a new Options value with ValueDir set to the given value.
361 //
362 // ValueDir is the path of the directory where value data will be stored in.
363 // If it doesn't exist, Badger will try to create it for you.
364 // This is set automatically to be the path given to `DefaultOptions`.
365 func (opt Options) WithValueDir(val string) Options {
366 opt.ValueDir = val
367 return opt
368 }
369
370 // WithSyncWrites returns a new Options value with SyncWrites set to the given value.
371 //
372 // Badger does all writes via mmap. So, all writes can survive process crashes or k8s environments
373 // with SyncWrites set to false.
374 //
375 // When set to true, Badger would call an additional msync after writes to flush mmap buffer over to
376 // disk to survive hard reboots. Most users of Badger should not need to do this.
377 //
378 // The default value of SyncWrites is false.
379 func (opt Options) WithSyncWrites(val bool) Options {
380 opt.SyncWrites = val
381 return opt
382 }
383
384 // WithNumVersionsToKeep returns a new Options value with NumVersionsToKeep set to the given value.
385 //
386 // NumVersionsToKeep sets how many versions to keep per key at most.
387 //
388 // The default value of NumVersionsToKeep is 1.
389 func (opt Options) WithNumVersionsToKeep(val int) Options {
390 opt.NumVersionsToKeep = val
391 return opt
392 }
393
394 // WithNumGoroutines sets the number of goroutines to be used in Stream.
395 //
396 // The default value of NumGoroutines is 8.
397 func (opt Options) WithNumGoroutines(val int) Options {
398 opt.NumGoroutines = val
399 return opt
400 }
401
402 // WithReadOnly returns a new Options value with ReadOnly set to the given value.
403 //
404 // When ReadOnly is true the DB will be opened on read-only mode.
405 // Multiple processes can open the same Badger DB.
406 // Note: if the DB being opened had crashed before and has vlog data to be replayed,
407 // ReadOnly will cause Open to fail with an appropriate message.
408 //
409 // The default value of ReadOnly is false.
410 func (opt Options) WithReadOnly(val bool) Options {
411 opt.ReadOnly = val
412 return opt
413 }
414
415 // WithMetricsEnabled returns a new Options value with MetricsEnabled set to the given value.
416 //
417 // When MetricsEnabled is set to false, then the DB will be opened and no badger metrics
418 // will be logged. Metrics are defined in metric.go file.
419 //
420 // This flag is useful for use cases like in Dgraph where we open temporary badger instances to
421 // index data. In those cases we don't want badger metrics to be polluted with the noise from
422 // those temporary instances.
423 //
424 // Default value is set to true
425 func (opt Options) WithMetricsEnabled(val bool) Options {
426 opt.MetricsEnabled = val
427 return opt
428 }
429
430 // WithLogger returns a new Options value with Logger set to the given value.
431 //
432 // Logger provides a way to configure what logger each value of badger.DB uses.
433 //
434 // The default value of Logger writes to stderr using the log package from the Go standard library.
435 func (opt Options) WithLogger(val Logger) Options {
436 opt.Logger = val
437 return opt
438 }
439
440 // WithLoggingLevel returns a new Options value with logging level of the
441 // default logger set to the given value.
442 // LoggingLevel sets the level of logging. It should be one of DEBUG, INFO,
443 // WARNING or ERROR levels.
444 //
445 // The default value of LoggingLevel is INFO.
446 func (opt Options) WithLoggingLevel(val loggingLevel) Options {
447 opt.Logger = defaultLogger(val)
448 return opt
449 }
450
451 // WithBaseTableSize returns a new Options value with BaseTableSize set to the given value.
452 //
453 // BaseTableSize sets the maximum size in bytes for LSM table or file in the base level.
454 //
455 // The default value of BaseTableSize is 2MB.
456 func (opt Options) WithBaseTableSize(val int64) Options {
457 opt.BaseTableSize = val
458 return opt
459 }
460
461 // WithLevelSizeMultiplier returns a new Options value with LevelSizeMultiplier set to the given
462 // value.
463 //
464 // LevelSizeMultiplier sets the ratio between the maximum sizes of contiguous levels in the LSM.
465 // Once a level grows to be larger than this ratio allowed, the compaction process will be
466 // triggered.
467 //
468 // The default value of LevelSizeMultiplier is 10.
469 func (opt Options) WithLevelSizeMultiplier(val int) Options {
470 opt.LevelSizeMultiplier = val
471 return opt
472 }
473
474 // WithMaxLevels returns a new Options value with MaxLevels set to the given value.
475 //
476 // Maximum number of levels of compaction allowed in the LSM.
477 //
478 // The default value of MaxLevels is 7.
479 func (opt Options) WithMaxLevels(val int) Options {
480 opt.MaxLevels = val
481 return opt
482 }
483
484 // WithValueThreshold returns a new Options value with ValueThreshold set to the given value.
485 //
486 // ValueThreshold sets the threshold used to decide whether a value is stored directly in the LSM
487 // tree or separately in the log value files.
488 //
489 // The default value of ValueThreshold is 1 MB, and LSMOnlyOptions sets it to maxValueThreshold
490 // which is set to 1 MB too.
491 func (opt Options) WithValueThreshold(val int64) Options {
492 opt.ValueThreshold = val
493 return opt
494 }
495
496 // WithVLogPercentile returns a new Options value with ValLogPercentile set to given value.
497 //
498 // VLogPercentile with 0.0 means no dynamic thresholding is enabled.
499 // MinThreshold value will always act as the value threshold.
500 //
501 // VLogPercentile with value 0.99 means 99 percentile of value will be put in LSM tree
502 // and only 1 percent in vlog. The value threshold will be dynamically updated within the range of
503 // [ValueThreshold, Options.maxValueThreshold]
504 //
505 // # Say VLogPercentile with 1.0 means threshold will eventually set to Options.maxValueThreshold
506 //
507 // The default value of VLogPercentile is 0.0.
508 func (opt Options) WithVLogPercentile(t float64) Options {
509 opt.VLogPercentile = t
510 return opt
511 }
512
513 // WithNumMemtables returns a new Options value with NumMemtables set to the given value.
514 //
515 // NumMemtables sets the maximum number of tables to keep in memory before stalling.
516 //
517 // The default value of NumMemtables is 5.
518 func (opt Options) WithNumMemtables(val int) Options {
519 opt.NumMemtables = val
520 return opt
521 }
522
523 // WithMemTableSize returns a new Options value with MemTableSize set to the given value.
524 //
525 // MemTableSize sets the maximum size in bytes for memtable table.
526 //
527 // The default value of MemTableSize is 64MB.
528 func (opt Options) WithMemTableSize(val int64) Options {
529 opt.MemTableSize = val
530 return opt
531 }
532
533 // WithBloomFalsePositive returns a new Options value with BloomFalsePositive set
534 // to the given value.
535 //
536 // BloomFalsePositive sets the false positive probability of the bloom filter in any SSTable.
537 // Before reading a key from table, the bloom filter is checked for key existence.
538 // BloomFalsePositive might impact read performance of DB. Lower BloomFalsePositive value might
539 // consume more memory.
540 //
541 // The default value of BloomFalsePositive is 0.01.
542 //
543 // Setting this to 0 disables the bloom filter completely.
544 func (opt Options) WithBloomFalsePositive(val float64) Options {
545 opt.BloomFalsePositive = val
546 return opt
547 }
548
549 // WithBlockSize returns a new Options value with BlockSize set to the given value.
550 //
551 // BlockSize sets the size of any block in SSTable. SSTable is divided into multiple blocks
552 // internally. Each block is compressed using prefix diff encoding.
553 //
554 // The default value of BlockSize is 4KB.
555 func (opt Options) WithBlockSize(val int) Options {
556 opt.BlockSize = val
557 return opt
558 }
559
560 // WithNumLevelZeroTables sets the maximum number of Level 0 tables before compaction starts.
561 //
562 // The default value of NumLevelZeroTables is 5.
563 func (opt Options) WithNumLevelZeroTables(val int) Options {
564 opt.NumLevelZeroTables = val
565 return opt
566 }
567
568 // WithNumLevelZeroTablesStall sets the number of Level 0 tables that once reached causes the DB to
569 // stall until compaction succeeds.
570 //
571 // The default value of NumLevelZeroTablesStall is 15.
572 func (opt Options) WithNumLevelZeroTablesStall(val int) Options {
573 opt.NumLevelZeroTablesStall = val
574 return opt
575 }
576
577 // WithBaseLevelSize sets the maximum size target for the base level.
578 //
579 // The default value is 10MB.
580 func (opt Options) WithBaseLevelSize(val int64) Options {
581 opt.BaseLevelSize = val
582 return opt
583 }
584
585 // WithValueLogFileSize sets the maximum size of a single value log file.
586 //
587 // The default value of ValueLogFileSize is 1GB.
588 func (opt Options) WithValueLogFileSize(val int64) Options {
589 opt.ValueLogFileSize = val
590 return opt
591 }
592
593 // WithValueLogMaxEntries sets the maximum number of entries a value log file
594 // can hold approximately. A actual size limit of a value log file is the
595 // minimum of ValueLogFileSize and ValueLogMaxEntries.
596 //
597 // The default value of ValueLogMaxEntries is one million (1000000).
598 func (opt Options) WithValueLogMaxEntries(val uint32) Options {
599 opt.ValueLogMaxEntries = val
600 return opt
601 }
602
603 // WithNumCompactors sets the number of compaction workers to run concurrently. Setting this to
604 // zero stops compactions, which could eventually cause writes to block forever.
605 //
606 // The default value of NumCompactors is 4. One is dedicated just for L0 and L1.
607 func (opt Options) WithNumCompactors(val int) Options {
608 opt.NumCompactors = val
609 return opt
610 }
611
612 // WithCompactL0OnClose determines whether Level 0 should be compacted before closing the DB. This
613 // ensures that both reads and writes are efficient when the DB is opened later.
614 //
615 // The default value of CompactL0OnClose is false.
616 func (opt Options) WithCompactL0OnClose(val bool) Options {
617 opt.CompactL0OnClose = val
618 return opt
619 }
620
621 // WithEncryptionKey is used to encrypt the data with AES. Type of AES is used based on the key
622 // size. For example 16 bytes will use AES-128. 24 bytes will use AES-192. 32 bytes will
623 // use AES-256.
624 func (opt Options) WithEncryptionKey(key []byte) Options {
625 opt.EncryptionKey = key
626 return opt
627 }
628
629 // WithEncryptionKeyRotationDuration returns new Options value with the duration set to
630 // the given value.
631 //
632 // Key Registry will use this duration to create new keys. If the previous generated
633 // key exceed the given duration. Then the key registry will create new key.
634
635 // The default value is set to 10 days.
636 func (opt Options) WithEncryptionKeyRotationDuration(d time.Duration) Options {
637 opt.EncryptionKeyRotationDuration = d
638 return opt
639 }
640
641 // WithCompression is used to enable or disable compression. When compression is enabled, every
642 // block will be compressed using the specified algorithm. This option doesn't affect existing
643 // tables. Only the newly created tables will be compressed.
644 //
645 // The default compression algorithm used is snappy. Compression is enabled by default.
646 func (opt Options) WithCompression(cType options.CompressionType) Options {
647 opt.Compression = cType
648 return opt
649 }
650
651 // WithVerifyValueChecksum is used to set VerifyValueChecksum. When VerifyValueChecksum is set to
652 // true, checksum will be verified for every entry read from the value log. If the value is stored
653 // in SST (value size less than value threshold) then the checksum validation will not be done.
654 //
655 // The default value of VerifyValueChecksum is False.
656 func (opt Options) WithVerifyValueChecksum(val bool) Options {
657 opt.VerifyValueChecksum = val
658 return opt
659 }
660
661 // WithChecksumVerificationMode returns a new Options value with ChecksumVerificationMode set to
662 // the given value.
663 //
664 // ChecksumVerificationMode indicates when the db should verify checksums for SSTable blocks.
665 //
666 // The default value of VerifyValueChecksum is options.NoVerification.
667 func (opt Options) WithChecksumVerificationMode(cvMode options.ChecksumVerificationMode) Options {
668 opt.ChecksumVerificationMode = cvMode
669 return opt
670 }
671
672 // WithBlockCacheSize returns a new Options value with BlockCacheSize set to the given value.
673 //
674 // This value specifies how much data cache should hold in memory. A small size
675 // of cache means lower memory consumption and lookups/iterations would take
676 // longer. It is recommended to use a cache if you're using compression or encryption.
677 // If compression and encryption both are disabled, adding a cache will lead to
678 // unnecessary overhead which will affect the read performance. Setting size to
679 // zero disables the cache altogether.
680 //
681 // Default value of BlockCacheSize is 256 MB.
682 func (opt Options) WithBlockCacheSize(size int64) Options {
683 opt.BlockCacheSize = size
684 return opt
685 }
686
687 // WithInMemory returns a new Options value with Inmemory mode set to the given value.
688 //
689 // When badger is running in InMemory mode, everything is stored in memory. No value/sst files are
690 // created. In case of a crash all data will be lost.
691 func (opt Options) WithInMemory(b bool) Options {
692 opt.InMemory = b
693 return opt
694 }
695
696 // WithZSTDCompressionLevel returns a new Options value with ZSTDCompressionLevel set
697 // to the given value.
698 //
699 // The ZSTD compression algorithm supports 20 compression levels. The higher the compression
700 // level, the better is the compression ratio but lower is the performance. Lower levels
701 // have better performance and higher levels have better compression ratios.
702 // We recommend using level 1 ZSTD Compression Level. Any level higher than 1 seems to
703 // deteriorate badger's performance.
704 // The following benchmarks were done on a 4 KB block size (default block size). The compression is
705 // ratio supposed to increase with increasing compression level but since the input for compression
706 // algorithm is small (4 KB), we don't get significant benefit at level 3. It is advised to write
707 // your own benchmarks before choosing a compression algorithm or level.
708 //
709 // NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid.
710 // no_compression-16 10 502848865 ns/op 165.46 MB/s -
711 // zstd_compression/level_1-16 7 739037966 ns/op 112.58 MB/s 2.93
712 // zstd_compression/level_3-16 7 756950250 ns/op 109.91 MB/s 2.72
713 // zstd_compression/level_15-16 1 11135686219 ns/op 7.47 MB/s 4.38
714 // Benchmark code can be found in table/builder_test.go file
715 func (opt Options) WithZSTDCompressionLevel(cLevel int) Options {
716 opt.ZSTDCompressionLevel = cLevel
717 return opt
718 }
719
720 // WithBypassLockGuard returns a new Options value with BypassLockGuard
721 // set to the given value.
722 //
723 // When BypassLockGuard option is set, badger will not acquire a lock on the
724 // directory. This could lead to data corruption if multiple badger instances
725 // write to the same data directory. Use this option with caution.
726 //
727 // The default value of BypassLockGuard is false.
728 func (opt Options) WithBypassLockGuard(b bool) Options {
729 opt.BypassLockGuard = b
730 return opt
731 }
732
733 // WithIndexCacheSize returns a new Options value with IndexCacheSize set to
734 // the given value.
735 //
736 // This value specifies how much memory should be used by table indices. These
737 // indices include the block offsets and the bloomfilters. Badger uses bloom
738 // filters to speed up lookups. Each table has its own bloom
739 // filter and each bloom filter is approximately of 5 MB.
740 //
741 // Zero value for IndexCacheSize means all the indices will be kept in
742 // memory and the cache is disabled.
743 //
744 // The default value of IndexCacheSize is 0 which means all indices are kept in
745 // memory.
746 func (opt Options) WithIndexCacheSize(size int64) Options {
747 opt.IndexCacheSize = size
748 return opt
749 }
750
751 // WithDetectConflicts returns a new Options value with DetectConflicts set to the given value.
752 //
753 // Detect conflicts options determines if the transactions would be checked for
754 // conflicts before committing them. When this option is set to false
755 // (detectConflicts=false) badger can process transactions at a higher rate.
756 // Setting this options to false might be useful when the user application
757 // deals with conflict detection and resolution.
758 //
759 // The default value of Detect conflicts is True.
760 func (opt Options) WithDetectConflicts(b bool) Options {
761 opt.DetectConflicts = b
762 return opt
763 }
764
765 // WithNamespaceOffset returns a new Options value with NamespaceOffset set to the given value. DB
766 // will expect the namespace in each key at the 8 bytes starting from NamespaceOffset. A negative
767 // value means that namespace is not stored in the key.
768 //
769 // The default value for NamespaceOffset is -1.
770 func (opt Options) WithNamespaceOffset(offset int) Options {
771 opt.NamespaceOffset = offset
772 return opt
773 }
774
775 // WithExternalMagic returns a new Options value with ExternalMagicVersion set to the given value.
776 // The DB would fail to start if either the internal or the external magic number fails validated.
777 func (opt Options) WithExternalMagic(magic uint16) Options {
778 opt.ExternalMagicVersion = magic
779 return opt
780 }
781
782 func (opt Options) getFileFlags() int {
783 var flags int
784 // opt.SyncWrites would be using msync to sync. All writes go through mmap.
785 if opt.ReadOnly {
786 flags |= os.O_RDONLY
787 } else {
788 flags |= os.O_RDWR
789 }
790 return flags
791 }
792