1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 package unicode
6 7 import (
8 "golang.org/x/text/transform"
9 )
10 11 // BOMOverride returns a new decoder transformer that is identical to fallback,
12 // except that the presence of a Byte Order Mark at the start of the input
13 // causes it to switch to the corresponding Unicode decoding. It will only
14 // consider BOMs for UTF-8, UTF-16BE, and UTF-16LE.
15 //
16 // This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not
17 // just UTF-16 variants, and allowing falling back to any encoding scheme.
18 //
19 // This technique is recommended by the W3C for use in HTML 5: "For
20 // compatibility with deployed content, the byte order mark (also known as BOM)
21 // is considered more authoritative than anything else."
22 // http://www.w3.org/TR/encoding/#specification-hooks
23 //
24 // Using BOMOverride is mostly intended for use cases where the first characters
25 // of a fallback encoding are known to not be a BOM, for example, for valid HTML
26 // and most encodings.
27 func BOMOverride(fallback transform.Transformer) transform.Transformer {
28 // TODO: possibly allow a variadic argument of unicode encodings to allow
29 // specifying details of which fallbacks are supported as well as
30 // specifying the details of the implementations. This would also allow for
31 // support for UTF-32, which should not be supported by default.
32 return &bomOverride{fallback: fallback}
33 }
34 35 type bomOverride struct {
36 fallback transform.Transformer
37 current transform.Transformer
38 }
39 40 func (d *bomOverride) Reset() {
41 d.current = nil
42 d.fallback.Reset()
43 }
44 45 var (
46 // TODO: we could use decode functions here, instead of allocating a new
47 // decoder on every NewDecoder as IgnoreBOM decoders can be stateless.
48 utf16le = UTF16(LittleEndian, IgnoreBOM)
49 utf16be = UTF16(BigEndian, IgnoreBOM)
50 )
51 52 const utf8BOM = "\ufeff"
53 54 func (d *bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
55 if d.current != nil {
56 return d.current.Transform(dst, src, atEOF)
57 }
58 if len(src) < 3 && !atEOF {
59 return 0, 0, transform.ErrShortSrc
60 }
61 d.current = d.fallback
62 bomSize := 0
63 if len(src) >= 2 {
64 if src[0] == 0xFF && src[1] == 0xFE {
65 d.current = utf16le.NewDecoder()
66 bomSize = 2
67 } else if src[0] == 0xFE && src[1] == 0xFF {
68 d.current = utf16be.NewDecoder()
69 bomSize = 2
70 } else if len(src) >= 3 &&
71 src[0] == utf8BOM[0] &&
72 src[1] == utf8BOM[1] &&
73 src[2] == utf8BOM[2] {
74 d.current = transform.Nop
75 bomSize = 3
76 }
77 }
78 if bomSize < len(src) {
79 nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF)
80 }
81 return nDst, nSrc + bomSize, err
82 }
83