1 package zstd
2 3 import (
4 "errors"
5 "fmt"
6 "math"
7 "math/bits"
8 "runtime"
9 "strings"
10 )
11 12 // EOption is an option for creating a encoder.
13 type EOption func(*encoderOptions) error
14 15 // options retains accumulated state of multiple options.
16 type encoderOptions struct {
17 concurrent int
18 level EncoderLevel
19 single *bool
20 pad int
21 blockSize int
22 windowSize int
23 crc bool
24 fullZero bool
25 noEntropy bool
26 allLitEntropy bool
27 customWindow bool
28 customALEntropy bool
29 customBlockSize bool
30 lowMem bool
31 dict *dict
32 }
33 34 func (o *encoderOptions) setDefault() {
35 *o = encoderOptions{
36 concurrent: runtime.GOMAXPROCS(0),
37 crc: true,
38 single: nil,
39 blockSize: maxCompressedBlockSize,
40 windowSize: 8 << 20,
41 level: SpeedDefault,
42 allLitEntropy: false,
43 lowMem: false,
44 }
45 }
46 47 // encoder returns an encoder with the selected options.
48 func (o encoderOptions) encoder() encoder {
49 switch o.level {
50 case SpeedFastest:
51 if o.dict != nil {
52 return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
53 }
54 return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
55 56 case SpeedDefault:
57 if o.dict != nil {
58 return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}}
59 }
60 return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
61 case SpeedBetterCompression:
62 if o.dict != nil {
63 return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
64 }
65 return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
66 case SpeedBestCompression:
67 return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
68 }
69 panic("unknown compression level")
70 }
71 72 // WithEncoderCRC will add CRC value to output.
73 // Output will be 4 bytes larger.
74 func WithEncoderCRC(b bool) EOption {
75 return func(o *encoderOptions) error { o.crc = b; return nil }
76 }
77 78 // WithEncoderConcurrency will set the concurrency,
79 // meaning the maximum number of encoders to run concurrently.
80 // The value supplied must be at least 1.
81 // For streams, setting a value of 1 will disable async compression.
82 // By default this will be set to GOMAXPROCS.
83 func WithEncoderConcurrency(n int) EOption {
84 return func(o *encoderOptions) error {
85 if n <= 0 {
86 return fmt.Errorf("concurrency must be at least 1")
87 }
88 o.concurrent = n
89 return nil
90 }
91 }
92 93 // WithWindowSize will set the maximum allowed back-reference distance.
94 // The value must be a power of two between MinWindowSize and MaxWindowSize.
95 // A larger value will enable better compression but allocate more memory and,
96 // for above-default values, take considerably longer.
97 // The default value is determined by the compression level and max 8MB.
98 func WithWindowSize(n int) EOption {
99 return func(o *encoderOptions) error {
100 switch {
101 case n < MinWindowSize:
102 return fmt.Errorf("window size must be at least %d", MinWindowSize)
103 case n > MaxWindowSize:
104 return fmt.Errorf("window size must be at most %d", MaxWindowSize)
105 case (n & (n - 1)) != 0:
106 return errors.New("window size must be a power of 2")
107 }
108 109 o.windowSize = n
110 o.customWindow = true
111 if o.blockSize > o.windowSize {
112 o.blockSize = o.windowSize
113 o.customBlockSize = true
114 }
115 return nil
116 }
117 }
118 119 // WithEncoderPadding will add padding to all output so the size will be a multiple of n.
120 // This can be used to obfuscate the exact output size or make blocks of a certain size.
121 // The contents will be a skippable frame, so it will be invisible by the decoder.
122 // n must be > 0 and <= 1GB, 1<<30 bytes.
123 // The padded area will be filled with data from crypto/rand.Reader.
124 // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
125 func WithEncoderPadding(n int) EOption {
126 return func(o *encoderOptions) error {
127 if n <= 0 {
128 return fmt.Errorf("padding must be at least 1")
129 }
130 // No need to waste our time.
131 if n == 1 {
132 n = 0
133 }
134 if n > 1<<30 {
135 return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ")
136 }
137 o.pad = n
138 return nil
139 }
140 }
141 142 // EncoderLevel predefines encoder compression levels.
143 // Only use the constants made available, since the actual mapping
144 // of these values are very likely to change and your compression could change
145 // unpredictably when upgrading the library.
146 type EncoderLevel int
147 148 const (
149 speedNotSet EncoderLevel = iota
150 151 // SpeedFastest will choose the fastest reasonable compression.
152 // This is roughly equivalent to the fastest Zstandard mode.
153 SpeedFastest
154 155 // SpeedDefault is the default "pretty fast" compression option.
156 // This is roughly equivalent to the default Zstandard mode (level 3).
157 SpeedDefault
158 159 // SpeedBetterCompression will yield better compression than the default.
160 // Currently it is about zstd level 7-8 with ~ 2x-3x the default CPU usage.
161 // By using this, notice that CPU usage may go up in the future.
162 SpeedBetterCompression
163 164 // SpeedBestCompression will choose the best available compression option.
165 // This will offer the best compression no matter the CPU cost.
166 SpeedBestCompression
167 168 // speedLast should be kept as the last actual compression option.
169 // The is not for external usage, but is used to keep track of the valid options.
170 speedLast
171 )
172 173 // EncoderLevelFromString will convert a string representation of an encoding level back
174 // to a compression level. The compare is not case sensitive.
175 // If the string wasn't recognized, (false, SpeedDefault) will be returned.
176 func EncoderLevelFromString(s string) (bool, EncoderLevel) {
177 for l := speedNotSet + 1; l < speedLast; l++ {
178 if strings.EqualFold(s, l.String()) {
179 return true, l
180 }
181 }
182 return false, SpeedDefault
183 }
184 185 // EncoderLevelFromZstd will return an encoder level that closest matches the compression
186 // ratio of a specific zstd compression level.
187 // Many input values will provide the same compression level.
188 func EncoderLevelFromZstd(level int) EncoderLevel {
189 switch {
190 case level < 3:
191 return SpeedFastest
192 case level >= 3 && level < 6:
193 return SpeedDefault
194 case level >= 6 && level < 10:
195 return SpeedBetterCompression
196 default:
197 return SpeedBestCompression
198 }
199 }
200 201 // String provides a string representation of the compression level.
202 func (e EncoderLevel) String() string {
203 switch e {
204 case SpeedFastest:
205 return "fastest"
206 case SpeedDefault:
207 return "default"
208 case SpeedBetterCompression:
209 return "better"
210 case SpeedBestCompression:
211 return "best"
212 default:
213 return "invalid"
214 }
215 }
216 217 // WithEncoderLevel specifies a predefined compression level.
218 func WithEncoderLevel(l EncoderLevel) EOption {
219 return func(o *encoderOptions) error {
220 switch {
221 case l <= speedNotSet || l >= speedLast:
222 return fmt.Errorf("unknown encoder level")
223 }
224 o.level = l
225 if !o.customWindow {
226 switch o.level {
227 case SpeedFastest:
228 o.windowSize = 4 << 20
229 if !o.customBlockSize {
230 o.blockSize = 1 << 16
231 }
232 case SpeedDefault:
233 o.windowSize = 8 << 20
234 case SpeedBetterCompression:
235 o.windowSize = 8 << 20
236 case SpeedBestCompression:
237 o.windowSize = 8 << 20
238 }
239 }
240 if !o.customALEntropy {
241 o.allLitEntropy = l > SpeedDefault
242 }
243 244 return nil
245 }
246 }
247 248 // WithZeroFrames will encode 0 length input as full frames.
249 // This can be needed for compatibility with zstandard usage,
250 // but is not needed for this package.
251 func WithZeroFrames(b bool) EOption {
252 return func(o *encoderOptions) error {
253 o.fullZero = b
254 return nil
255 }
256 }
257 258 // WithAllLitEntropyCompression will apply entropy compression if no matches are found.
259 // Disabling this will skip incompressible data faster, but in cases with no matches but
260 // skewed character distribution compression is lost.
261 // Default value depends on the compression level selected.
262 func WithAllLitEntropyCompression(b bool) EOption {
263 return func(o *encoderOptions) error {
264 o.customALEntropy = true
265 o.allLitEntropy = b
266 return nil
267 }
268 }
269 270 // WithNoEntropyCompression will always skip entropy compression of literals.
271 // This can be useful if content has matches, but unlikely to benefit from entropy
272 // compression. Usually the slight speed improvement is not worth enabling this.
273 func WithNoEntropyCompression(b bool) EOption {
274 return func(o *encoderOptions) error {
275 o.noEntropy = b
276 return nil
277 }
278 }
279 280 // WithSingleSegment will set the "single segment" flag when EncodeAll is used.
281 // If this flag is set, data must be regenerated within a single continuous memory segment.
282 // In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present.
283 // As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content.
284 // In order to preserve the decoder from unreasonable memory requirements,
285 // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range.
286 // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB.
287 // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
288 // If this is not specified, block encodes will automatically choose this based on the input size and the window size.
289 // This setting has no effect on streamed encodes.
290 func WithSingleSegment(b bool) EOption {
291 return func(o *encoderOptions) error {
292 o.single = &b
293 return nil
294 }
295 }
296 297 // WithLowerEncoderMem will trade in some memory cases trade less memory usage for
298 // slower encoding speed.
299 // This will not change the window size which is the primary function for reducing
300 // memory usage. See WithWindowSize.
301 func WithLowerEncoderMem(b bool) EOption {
302 return func(o *encoderOptions) error {
303 o.lowMem = b
304 return nil
305 }
306 }
307 308 // WithEncoderDict allows to register a dictionary that will be used for the encode.
309 //
310 // The slice dict must be in the [dictionary format] produced by
311 // "zstd --train" from the Zstandard reference implementation.
312 //
313 // The encoder *may* choose to use no dictionary instead for certain payloads.
314 //
315 // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
316 func WithEncoderDict(dict []byte) EOption {
317 return func(o *encoderOptions) error {
318 d, err := loadDict(dict)
319 if err != nil {
320 return err
321 }
322 o.dict = d
323 return nil
324 }
325 }
326 327 // WithEncoderDictRaw registers a dictionary that may be used by the encoder.
328 //
329 // The slice content may contain arbitrary data. It will be used as an initial
330 // history.
331 func WithEncoderDictRaw(id uint32, content []byte) EOption {
332 return func(o *encoderOptions) error {
333 if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
334 return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
335 }
336 o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}
337 return nil
338 }
339 }
340