1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 //go:generate go run gen.go -output tables.go
6 7 package language
8 9 // TODO: Remove above NOTE after:
10 // - verifying that tables are dropped correctly (most notably matcher tables).
11 12 import (
13 "strings"
14 15 "golang.org/x/text/internal/language"
16 "golang.org/x/text/internal/language/compact"
17 )
18 19 // Tag represents a BCP 47 language tag. It is used to specify an instance of a
20 // specific language or locale. All language tag values are guaranteed to be
21 // well-formed.
22 type Tag compact.Tag
23 24 func makeTag(t language.Tag) (tag Tag) {
25 return Tag(compact.Make(t))
26 }
27 28 func (t *Tag) tag() language.Tag {
29 return (*compact.Tag)(t).Tag()
30 }
31 32 func (t *Tag) isCompact() bool {
33 return (*compact.Tag)(t).IsCompact()
34 }
35 36 // TODO: improve performance.
37 func (t *Tag) lang() language.Language { return t.tag().LangID }
38 func (t *Tag) region() language.Region { return t.tag().RegionID }
39 func (t *Tag) script() language.Script { return t.tag().ScriptID }
40 41 // Make is a convenience wrapper for Parse that omits the error.
42 // In case of an error, a sensible default is returned.
43 func Make(s string) Tag {
44 return Default.Make(s)
45 }
46 47 // Make is a convenience wrapper for c.Parse that omits the error.
48 // In case of an error, a sensible default is returned.
49 func (c CanonType) Make(s string) Tag {
50 t, _ := c.Parse(s)
51 return t
52 }
53 54 // Raw returns the raw base language, script and region, without making an
55 // attempt to infer their values.
56 func (t Tag) Raw() (b Base, s Script, r Region) {
57 tt := t.tag()
58 return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
59 }
60 61 // IsRoot returns true if t is equal to language "und".
62 func (t Tag) IsRoot() bool {
63 return compact.Tag(t).IsRoot()
64 }
65 66 // CanonType can be used to enable or disable various types of canonicalization.
67 type CanonType int
68 69 const (
70 // Replace deprecated base languages with their preferred replacements.
71 DeprecatedBase CanonType = 1 << iota
72 // Replace deprecated scripts with their preferred replacements.
73 DeprecatedScript
74 // Replace deprecated regions with their preferred replacements.
75 DeprecatedRegion
76 // Remove redundant scripts.
77 SuppressScript
78 // Normalize legacy encodings. This includes legacy languages defined in
79 // CLDR as well as bibliographic codes defined in ISO-639.
80 Legacy
81 // Map the dominant language of a macro language group to the macro language
82 // subtag. For example cmn -> zh.
83 Macro
84 // The CLDR flag should be used if full compatibility with CLDR is required.
85 // There are a few cases where language.Tag may differ from CLDR. To follow all
86 // of CLDR's suggestions, use All|CLDR.
87 CLDR
88 89 // Raw can be used to Compose or Parse without Canonicalization.
90 Raw CanonType = 0
91 92 // Replace all deprecated tags with their preferred replacements.
93 Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
94 95 // All canonicalizations recommended by BCP 47.
96 BCP47 = Deprecated | SuppressScript
97 98 // All canonicalizations.
99 All = BCP47 | Legacy | Macro
100 101 // Default is the canonicalization used by Parse, Make and Compose. To
102 // preserve as much information as possible, canonicalizations that remove
103 // potentially valuable information are not included. The Matcher is
104 // designed to recognize similar tags that would be the same if
105 // they were canonicalized using All.
106 Default = Deprecated | Legacy
107 108 canonLang = DeprecatedBase | Legacy | Macro
109 110 // TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
111 )
112 113 // canonicalize returns the canonicalized equivalent of the tag and
114 // whether there was any change.
115 func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
116 if c == Raw {
117 return t, false
118 }
119 changed := false
120 if c&SuppressScript != 0 {
121 if t.LangID.SuppressScript() == t.ScriptID {
122 t.ScriptID = 0
123 changed = true
124 }
125 }
126 if c&canonLang != 0 {
127 for {
128 if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
129 switch aliasType {
130 case language.Legacy:
131 if c&Legacy != 0 {
132 if t.LangID == _sh && t.ScriptID == 0 {
133 t.ScriptID = _Latn
134 }
135 t.LangID = l
136 changed = true
137 }
138 case language.Macro:
139 if c&Macro != 0 {
140 // We deviate here from CLDR. The mapping "nb" -> "no"
141 // qualifies as a typical Macro language mapping. However,
142 // for legacy reasons, CLDR maps "no", the macro language
143 // code for Norwegian, to the dominant variant "nb". This
144 // change is currently under consideration for CLDR as well.
145 // See https://unicode.org/cldr/trac/ticket/2698 and also
146 // https://unicode.org/cldr/trac/ticket/1790 for some of the
147 // practical implications. TODO: this check could be removed
148 // if CLDR adopts this change.
149 if c&CLDR == 0 || t.LangID != _nb {
150 changed = true
151 t.LangID = l
152 }
153 }
154 case language.Deprecated:
155 if c&DeprecatedBase != 0 {
156 if t.LangID == _mo && t.RegionID == 0 {
157 t.RegionID = _MD
158 }
159 t.LangID = l
160 changed = true
161 // Other canonicalization types may still apply.
162 continue
163 }
164 }
165 } else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
166 t.LangID = _nb
167 changed = true
168 }
169 break
170 }
171 }
172 if c&DeprecatedScript != 0 {
173 if t.ScriptID == _Qaai {
174 changed = true
175 t.ScriptID = _Zinh
176 }
177 }
178 if c&DeprecatedRegion != 0 {
179 if r := t.RegionID.Canonicalize(); r != t.RegionID {
180 changed = true
181 t.RegionID = r
182 }
183 }
184 return t, changed
185 }
186 187 // Canonicalize returns the canonicalized equivalent of the tag.
188 func (c CanonType) Canonicalize(t Tag) (Tag, error) {
189 // First try fast path.
190 if t.isCompact() {
191 if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
192 return t, nil
193 }
194 }
195 // It is unlikely that one will canonicalize a tag after matching. So do
196 // a slow but simple approach here.
197 if tag, changed := canonicalize(c, t.tag()); changed {
198 tag.RemakeString()
199 return makeTag(tag), nil
200 }
201 return t, nil
202 203 }
204 205 // Confidence indicates the level of certainty for a given return value.
206 // For example, Serbian may be written in Cyrillic or Latin script.
207 // The confidence level indicates whether a value was explicitly specified,
208 // whether it is typically the only possible value, or whether there is
209 // an ambiguity.
210 type Confidence int
211 212 const (
213 No Confidence = iota // full confidence that there was no match
214 Low // most likely value picked out of a set of alternatives
215 High // value is generally assumed to be the correct match
216 Exact // exact match or explicitly specified value
217 )
218 219 var confName = []string{"No", "Low", "High", "Exact"}
220 221 func (c Confidence) String() string {
222 return confName[c]
223 }
224 225 // String returns the canonical string representation of the language tag.
226 func (t Tag) String() string {
227 return t.tag().String()
228 }
229 230 // MarshalText implements encoding.TextMarshaler.
231 func (t Tag) MarshalText() (text []byte, err error) {
232 return t.tag().MarshalText()
233 }
234 235 // UnmarshalText implements encoding.TextUnmarshaler.
236 func (t *Tag) UnmarshalText(text []byte) error {
237 var tag language.Tag
238 err := tag.UnmarshalText(text)
239 *t = makeTag(tag)
240 return err
241 }
242 243 // Base returns the base language of the language tag. If the base language is
244 // unspecified, an attempt will be made to infer it from the context.
245 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
246 func (t Tag) Base() (Base, Confidence) {
247 if b := t.lang(); b != 0 {
248 return Base{b}, Exact
249 }
250 tt := t.tag()
251 c := High
252 if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
253 c = Low
254 }
255 if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
256 return Base{tag.LangID}, c
257 }
258 return Base{0}, No
259 }
260 261 // Script infers the script for the language tag. If it was not explicitly given, it will infer
262 // a most likely candidate.
263 // If more than one script is commonly used for a language, the most likely one
264 // is returned with a low confidence indication. For example, it returns (Cyrl, Low)
265 // for Serbian.
266 // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
267 // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
268 // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
269 // See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
270 // unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified.
271 // Note that an inferred script is never guaranteed to be the correct one. Latin is
272 // almost exclusively used for Afrikaans, but Arabic has been used for some texts
273 // in the past. Also, the script that is commonly used may change over time.
274 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
275 func (t Tag) Script() (Script, Confidence) {
276 if scr := t.script(); scr != 0 {
277 return Script{scr}, Exact
278 }
279 tt := t.tag()
280 sc, c := language.Script(_Zzzz), No
281 if scr := tt.LangID.SuppressScript(); scr != 0 {
282 // Note: it is not always the case that a language with a suppress
283 // script value is only written in one script (e.g. kk, ms, pa).
284 if tt.RegionID == 0 {
285 return Script{scr}, High
286 }
287 sc, c = scr, High
288 }
289 if tag, err := tt.Maximize(); err == nil {
290 if tag.ScriptID != sc {
291 sc, c = tag.ScriptID, Low
292 }
293 } else {
294 tt, _ = canonicalize(Deprecated|Macro, tt)
295 if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
296 sc, c = tag.ScriptID, Low
297 }
298 }
299 return Script{sc}, c
300 }
301 302 // Region returns the region for the language tag. If it was not explicitly given, it will
303 // infer a most likely candidate from the context.
304 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
305 func (t Tag) Region() (Region, Confidence) {
306 if r := t.region(); r != 0 {
307 return Region{r}, Exact
308 }
309 tt := t.tag()
310 if tt, err := tt.Maximize(); err == nil {
311 return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
312 }
313 tt, _ = canonicalize(Deprecated|Macro, tt)
314 if tag, err := tt.Maximize(); err == nil {
315 return Region{tag.RegionID}, Low
316 }
317 return Region{_ZZ}, No // TODO: return world instead of undetermined?
318 }
319 320 // Variants returns the variants specified explicitly for this language tag.
321 // or nil if no variant was specified.
322 func (t Tag) Variants() []Variant {
323 if !compact.Tag(t).MayHaveVariants() {
324 return nil
325 }
326 v := []Variant{}
327 x, str := "", t.tag().Variants()
328 for str != "" {
329 x, str = nextToken(str)
330 v = append(v, Variant{x})
331 }
332 return v
333 }
334 335 // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
336 // specific language are substituted with fields from the parent language.
337 // The parent for a language may change for newer versions of CLDR.
338 //
339 // Parent returns a tag for a less specific language that is mutually
340 // intelligible or Und if there is no such language. This may not be the same as
341 // simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
342 // is "zh-Hant", and the parent of "zh-Hant" is "und".
343 func (t Tag) Parent() Tag {
344 return Tag(compact.Tag(t).Parent())
345 }
346 347 // nextToken returns token t and the rest of the string.
348 func nextToken(s string) (t, tail string) {
349 p := strings.Index(s[1:], "-")
350 if p == -1 {
351 return s[1:], ""
352 }
353 p++
354 return s[1:p], s[p:]
355 }
356 357 // Extension is a single BCP 47 extension.
358 type Extension struct {
359 s string
360 }
361 362 // String returns the string representation of the extension, including the
363 // type tag.
364 func (e Extension) String() string {
365 return e.s
366 }
367 368 // ParseExtension parses s as an extension and returns it on success.
369 func ParseExtension(s string) (e Extension, err error) {
370 ext, err := language.ParseExtension(s)
371 return Extension{ext}, err
372 }
373 374 // Type returns the one-byte extension type of e. It returns 0 for the zero
375 // exception.
376 func (e Extension) Type() byte {
377 if e.s == "" {
378 return 0
379 }
380 return e.s[0]
381 }
382 383 // Tokens returns the list of tokens of e.
384 func (e Extension) Tokens() []string {
385 return strings.Split(e.s, "-")
386 }
387 388 // Extension returns the extension of type x for tag t. It will return
389 // false for ok if t does not have the requested extension. The returned
390 // extension will be invalid in this case.
391 func (t Tag) Extension(x byte) (ext Extension, ok bool) {
392 if !compact.Tag(t).MayHaveExtensions() {
393 return Extension{}, false
394 }
395 e, ok := t.tag().Extension(x)
396 return Extension{e}, ok
397 }
398 399 // Extensions returns all extensions of t.
400 func (t Tag) Extensions() []Extension {
401 if !compact.Tag(t).MayHaveExtensions() {
402 return nil
403 }
404 e := []Extension{}
405 for _, ext := range t.tag().Extensions() {
406 e = append(e, Extension{ext})
407 }
408 return e
409 }
410 411 // TypeForKey returns the type associated with the given key, where key and type
412 // are of the allowed values defined for the Unicode locale extension ('u') in
413 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
414 // TypeForKey will traverse the inheritance chain to get the correct value.
415 //
416 // If there are multiple types associated with a key, only the first will be
417 // returned. If there is no type associated with a key, it returns the empty
418 // string.
419 func (t Tag) TypeForKey(key string) string {
420 if !compact.Tag(t).MayHaveExtensions() {
421 if key != "rg" && key != "va" {
422 return ""
423 }
424 }
425 return t.tag().TypeForKey(key)
426 }
427 428 // SetTypeForKey returns a new Tag with the key set to type, where key and type
429 // are of the allowed values defined for the Unicode locale extension ('u') in
430 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
431 // An empty value removes an existing pair with the same key.
432 func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
433 tt, err := t.tag().SetTypeForKey(key, value)
434 return makeTag(tt), err
435 }
436 437 // NumCompactTags is the number of compact tags. The maximum tag is
438 // NumCompactTags-1.
439 const NumCompactTags = compact.NumCompactTags
440 441 // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
442 // for which data exists in the text repository.The index will change over time
443 // and should not be stored in persistent storage. If t does not match a compact
444 // index, exact will be false and the compact index will be returned for the
445 // first match after repeatedly taking the Parent of t.
446 func CompactIndex(t Tag) (index int, exact bool) {
447 id, exact := compact.LanguageID(compact.Tag(t))
448 return int(id), exact
449 }
450 451 var root = language.Tag{}
452 453 // Base is an ISO 639 language code, used for encoding the base language
454 // of a language tag.
455 type Base struct {
456 langID language.Language
457 }
458 459 // ParseBase parses a 2- or 3-letter ISO 639 code.
460 // It returns a ValueError if s is a well-formed but unknown language identifier
461 // or another error if another error occurred.
462 func ParseBase(s string) (Base, error) {
463 l, err := language.ParseBase(s)
464 return Base{l}, err
465 }
466 467 // String returns the BCP 47 representation of the base language.
468 func (b Base) String() string {
469 return b.langID.String()
470 }
471 472 // ISO3 returns the ISO 639-3 language code.
473 func (b Base) ISO3() string {
474 return b.langID.ISO3()
475 }
476 477 // IsPrivateUse reports whether this language code is reserved for private use.
478 func (b Base) IsPrivateUse() bool {
479 return b.langID.IsPrivateUse()
480 }
481 482 // Script is a 4-letter ISO 15924 code for representing scripts.
483 // It is idiomatically represented in title case.
484 type Script struct {
485 scriptID language.Script
486 }
487 488 // ParseScript parses a 4-letter ISO 15924 code.
489 // It returns a ValueError if s is a well-formed but unknown script identifier
490 // or another error if another error occurred.
491 func ParseScript(s string) (Script, error) {
492 sc, err := language.ParseScript(s)
493 return Script{sc}, err
494 }
495 496 // String returns the script code in title case.
497 // It returns "Zzzz" for an unspecified script.
498 func (s Script) String() string {
499 return s.scriptID.String()
500 }
501 502 // IsPrivateUse reports whether this script code is reserved for private use.
503 func (s Script) IsPrivateUse() bool {
504 return s.scriptID.IsPrivateUse()
505 }
506 507 // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
508 type Region struct {
509 regionID language.Region
510 }
511 512 // EncodeM49 returns the Region for the given UN M.49 code.
513 // It returns an error if r is not a valid code.
514 func EncodeM49(r int) (Region, error) {
515 rid, err := language.EncodeM49(r)
516 return Region{rid}, err
517 }
518 519 // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
520 // It returns a ValueError if s is a well-formed but unknown region identifier
521 // or another error if another error occurred.
522 func ParseRegion(s string) (Region, error) {
523 r, err := language.ParseRegion(s)
524 return Region{r}, err
525 }
526 527 // String returns the BCP 47 representation for the region.
528 // It returns "ZZ" for an unspecified region.
529 func (r Region) String() string {
530 return r.regionID.String()
531 }
532 533 // ISO3 returns the 3-letter ISO code of r.
534 // Note that not all regions have a 3-letter ISO code.
535 // In such cases this method returns "ZZZ".
536 func (r Region) ISO3() string {
537 return r.regionID.ISO3()
538 }
539 540 // M49 returns the UN M.49 encoding of r, or 0 if this encoding
541 // is not defined for r.
542 func (r Region) M49() int {
543 return r.regionID.M49()
544 }
545 546 // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
547 // may include private-use tags that are assigned by CLDR and used in this
548 // implementation. So IsPrivateUse and IsCountry can be simultaneously true.
549 func (r Region) IsPrivateUse() bool {
550 return r.regionID.IsPrivateUse()
551 }
552 553 // IsCountry returns whether this region is a country or autonomous area. This
554 // includes non-standard definitions from CLDR.
555 func (r Region) IsCountry() bool {
556 return r.regionID.IsCountry()
557 }
558 559 // IsGroup returns whether this region defines a collection of regions. This
560 // includes non-standard definitions from CLDR.
561 func (r Region) IsGroup() bool {
562 return r.regionID.IsGroup()
563 }
564 565 // Contains returns whether Region c is contained by Region r. It returns true
566 // if c == r.
567 func (r Region) Contains(c Region) bool {
568 return r.regionID.Contains(c.regionID)
569 }
570 571 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
572 // In all other cases it returns either the region itself or an error.
573 //
574 // This method may return an error for a region for which there exists a
575 // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
576 // region will already be canonicalized it was obtained from a Tag that was
577 // obtained using any of the default methods.
578 func (r Region) TLD() (Region, error) {
579 tld, err := r.regionID.TLD()
580 return Region{tld}, err
581 }
582 583 // Canonicalize returns the region or a possible replacement if the region is
584 // deprecated. It will not return a replacement for deprecated regions that
585 // are split into multiple regions.
586 func (r Region) Canonicalize() Region {
587 return Region{r.regionID.Canonicalize()}
588 }
589 590 // Variant represents a registered variant of a language as defined by BCP 47.
591 type Variant struct {
592 variant string
593 }
594 595 // ParseVariant parses and returns a Variant. An error is returned if s is not
596 // a valid variant.
597 func ParseVariant(s string) (Variant, error) {
598 v, err := language.ParseVariant(s)
599 return Variant{v.String()}, err
600 }
601 602 // String returns the string representation of the variant.
603 func (v Variant) String() string {
604 return v.variant
605 }
606