1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 package flate
6 7 import (
8 "math"
9 "math/bits"
10 "sort"
11 )
12 13 // hcode is a huffman code with a bit code and bit length.
14 type hcode struct {
15 code, len uint16
16 }
17 18 type huffmanEncoder struct {
19 codes []hcode
20 freqcache []literalNode
21 bitCount [17]int32
22 lns byLiteral // stored to avoid repeated allocation in generate
23 lfs byFreq // stored to avoid repeated allocation in generate
24 }
25 26 type literalNode struct {
27 literal uint16
28 freq int32
29 }
30 31 // A levelInfo describes the state of the constructed tree for a given depth.
32 type levelInfo struct {
33 // Our level. for better printing
34 level int32
35 36 // The frequency of the last node at this level
37 lastFreq int32
38 39 // The frequency of the next character to add to this level
40 nextCharFreq int32
41 42 // The frequency of the next pair (from level below) to add to this level.
43 // Only valid if the "needed" value of the next lower level is 0.
44 nextPairFreq int32
45 46 // The number of chains remaining to generate for this level before moving
47 // up to the next level
48 needed int32
49 }
50 51 // set sets the code and length of an hcode.
52 func (h *hcode) set(code uint16, length uint16) {
53 h.len = length
54 h.code = code
55 }
56 57 func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
58 59 func newHuffmanEncoder(size int) *huffmanEncoder {
60 return &huffmanEncoder{codes: []hcode{:size}}
61 }
62 63 // Generates a HuffmanCode corresponding to the fixed literal table.
64 func generateFixedLiteralEncoding() *huffmanEncoder {
65 h := newHuffmanEncoder(maxNumLit)
66 codes := h.codes
67 var ch uint16
68 for ch = 0; ch < maxNumLit; ch++ {
69 var bits uint16
70 var size uint16
71 switch {
72 case ch < 144:
73 // size 8, 000110000 .. 10111111
74 bits = ch + 48
75 size = 8
76 case ch < 256:
77 // size 9, 110010000 .. 111111111
78 bits = ch + 400 - 144
79 size = 9
80 case ch < 280:
81 // size 7, 0000000 .. 0010111
82 bits = ch - 256
83 size = 7
84 default:
85 // size 8, 11000000 .. 11000111
86 bits = ch + 192 - 280
87 size = 8
88 }
89 codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
90 }
91 return h
92 }
93 94 func generateFixedOffsetEncoding() *huffmanEncoder {
95 h := newHuffmanEncoder(30)
96 codes := h.codes
97 for ch := range codes {
98 codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
99 }
100 return h
101 }
102 103 var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
104 var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
105 106 func (h *huffmanEncoder) bitLength(freq []int32) int {
107 var total int
108 for i, f := range freq {
109 if f != 0 {
110 total += int(f) * int(h.codes[i].len)
111 }
112 }
113 return total
114 }
115 116 const maxBitsLimit = 16
117 118 // bitCounts computes the number of literals assigned to each bit size in the Huffman encoding.
119 // It is only called when list.length >= 3.
120 // The cases of 0, 1, and 2 literals are handled by special case code.
121 //
122 // list is an array of the literals with non-zero frequencies
123 // and their associated frequencies. The array is in order of increasing
124 // frequency and has as its last element a special element with frequency
125 // MaxInt32.
126 //
127 // maxBits is the maximum number of bits that should be used to encode any literal.
128 // It must be less than 16.
129 //
130 // bitCounts returns an integer slice in which slice[i] indicates the number of literals
131 // that should be encoded in i bits.
132 func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
133 if maxBits >= maxBitsLimit {
134 panic("flate: maxBits too large")
135 }
136 n := int32(len(list))
137 list = list[0 : n+1]
138 list[n] = maxNode()
139 140 // The tree can't have greater depth than n - 1, no matter what. This
141 // saves a little bit of work in some small cases
142 if maxBits > n-1 {
143 maxBits = n - 1
144 }
145 146 // Create information about each of the levels.
147 // A bogus "Level 0" whose sole purpose is so that
148 // level1.prev.needed==0. This makes level1.nextPairFreq
149 // be a legitimate value that never gets chosen.
150 var levels [maxBitsLimit]levelInfo
151 // leafCounts[i] counts the number of literals at the left
152 // of ancestors of the rightmost node at level i.
153 // leafCounts[i][j] is the number of literals at the left
154 // of the level j ancestor.
155 var leafCounts [maxBitsLimit][maxBitsLimit]int32
156 157 for level := int32(1); level <= maxBits; level++ {
158 // For every level, the first two items are the first two characters.
159 // We initialize the levels as if we had already figured this out.
160 levels[level] = levelInfo{
161 level: level,
162 lastFreq: list[1].freq,
163 nextCharFreq: list[2].freq,
164 nextPairFreq: list[0].freq + list[1].freq,
165 }
166 leafCounts[level][level] = 2
167 if level == 1 {
168 levels[level].nextPairFreq = math.MaxInt32
169 }
170 }
171 172 // We need a total of 2*n - 2 items at top level and have already generated 2.
173 levels[maxBits].needed = 2*n - 4
174 175 level := maxBits
176 for {
177 l := &levels[level]
178 if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
179 // We've run out of both leaves and pairs.
180 // End all calculations for this level.
181 // To make sure we never come back to this level or any lower level,
182 // set nextPairFreq impossibly large.
183 l.needed = 0
184 levels[level+1].nextPairFreq = math.MaxInt32
185 level++
186 continue
187 }
188 189 prevFreq := l.lastFreq
190 if l.nextCharFreq < l.nextPairFreq {
191 // The next item on this row is a leaf node.
192 n := leafCounts[level][level] + 1
193 l.lastFreq = l.nextCharFreq
194 // Lower leafCounts are the same of the previous node.
195 leafCounts[level][level] = n
196 l.nextCharFreq = list[n].freq
197 } else {
198 // The next item on this row is a pair from the previous row.
199 // nextPairFreq isn't valid until we generate two
200 // more values in the level below
201 l.lastFreq = l.nextPairFreq
202 // Take leaf counts from the lower level, except counts[level] remains the same.
203 copy(leafCounts[level][:level], leafCounts[level-1][:level])
204 levels[l.level-1].needed = 2
205 }
206 207 if l.needed--; l.needed == 0 {
208 // We've done everything we need to do for this level.
209 // Continue calculating one level up. Fill in nextPairFreq
210 // of that level with the sum of the two nodes we've just calculated on
211 // this level.
212 if l.level == maxBits {
213 // All done!
214 break
215 }
216 levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
217 level++
218 } else {
219 // If we stole from below, move down temporarily to replenish it.
220 for levels[level-1].needed > 0 {
221 level--
222 }
223 }
224 }
225 226 // Somethings is wrong if at the end, the top level is null or hasn't used
227 // all of the leaves.
228 if leafCounts[maxBits][maxBits] != n {
229 panic("leafCounts[maxBits][maxBits] != n")
230 }
231 232 bitCount := h.bitCount[:maxBits+1]
233 bits := 1
234 counts := &leafCounts[maxBits]
235 for level := maxBits; level > 0; level-- {
236 // chain.leafCount gives the number of literals requiring at least "bits"
237 // bits to encode.
238 bitCount[bits] = counts[level] - counts[level-1]
239 bits++
240 }
241 return bitCount
242 }
243 244 // Look at the leaves and assign them a bit count and an encoding as specified
245 // in RFC 1951 3.2.2
246 func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
247 code := uint16(0)
248 for n, bits := range bitCount {
249 code <<= 1
250 if n == 0 || bits == 0 {
251 continue
252 }
253 // The literals list[len(list)-bits] .. list[len(list)-bits]
254 // are encoded using "bits" bits, and get the values
255 // code, code + 1, .... The code values are
256 // assigned in literal order (not frequency order).
257 chunk := list[len(list)-int(bits):]
258 259 h.lns.sort(chunk)
260 for _, node := range chunk {
261 h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
262 code++
263 }
264 list = list[0 : len(list)-int(bits)]
265 }
266 }
267 268 // Update this Huffman Code object to be the minimum code for the specified frequency count.
269 //
270 // freq is an array of frequencies, in which freq[i] gives the frequency of literal i.
271 // maxBits The maximum number of bits to use for any literal.
272 func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
273 if h.freqcache == nil {
274 // Allocate a reusable buffer with the longest possible frequency table.
275 // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
276 // The largest of these is maxNumLit, so we allocate for that case.
277 h.freqcache = []literalNode{:maxNumLit+1}
278 }
279 list := h.freqcache[:len(freq)+1]
280 // Number of non-zero literals
281 count := 0
282 // Set list to be the set of all non-zero literals and their frequencies
283 for i, f := range freq {
284 if f != 0 {
285 list[count] = literalNode{uint16(i), f}
286 count++
287 } else {
288 h.codes[i].len = 0
289 }
290 }
291 292 list = list[:count]
293 if count <= 2 {
294 // Handle the small cases here, because they are awkward for the general case code. With
295 // two or fewer literals, everything has bit length 1.
296 for i, node := range list {
297 // "list" is in order of increasing literal value.
298 h.codes[node.literal].set(uint16(i), 1)
299 }
300 return
301 }
302 h.lfs.sort(list)
303 304 // Get the number of literals for each bit count
305 bitCount := h.bitCounts(list, maxBits)
306 // And do the assignment
307 h.assignEncodingAndSize(bitCount, list)
308 }
309 310 type byLiteral []literalNode
311 312 func (s *byLiteral) sort(a []literalNode) {
313 *s = byLiteral(a)
314 sort.Sort(s)
315 }
316 317 func (s byLiteral) Len() int { return len(s) }
318 319 func (s byLiteral) Less(i, j int) bool {
320 return s[i].literal < s[j].literal
321 }
322 323 func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
324 325 type byFreq []literalNode
326 327 func (s *byFreq) sort(a []literalNode) {
328 *s = byFreq(a)
329 sort.Sort(s)
330 }
331 332 func (s byFreq) Len() int { return len(s) }
333 334 func (s byFreq) Less(i, j int) bool {
335 if s[i].freq == s[j].freq {
336 return s[i].literal < s[j].literal
337 }
338 return s[i].freq < s[j].freq
339 }
340 341 func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
342 343 func reverseBits(number uint16, bitLength byte) uint16 {
344 return bits.Reverse16(number << (16 - bitLength))
345 }
346