Checking in vendor folder for ease of using go get.

vendor/golang.org/x/text/encoding/ianaindex/example_test.go

@@ -0,0 +1,27 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ianaindex_test
+
+import (
+	"fmt"
+
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/ianaindex"
+)
+
+func ExampleIndex() {
+	fmt.Println(ianaindex.MIME.Name(charmap.ISO8859_7))
+	fmt.Println(ianaindex.IANA.Name(charmap.ISO8859_7))
+	fmt.Println(ianaindex.MIB.Name(charmap.ISO8859_7))
+
+	e, _ := ianaindex.IANA.Encoding("cp437")
+	fmt.Println(ianaindex.IANA.Name(e))
+
+	// Output:
+	// ISO-8859-7 <nil>
+	// ISO_8859-7:1987 <nil>
+	// ISOLatinGreek <nil>
+	// IBM437 <nil>
+}

vendor/golang.org/x/text/encoding/ianaindex/gen.go

@@ -0,0 +1,192 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/internal/gen"
+)
+
+type registry struct {
+	XMLName  xml.Name `xml:"registry"`
+	Updated  string   `xml:"updated"`
+	Registry []struct {
+		ID     string `xml:"id,attr"`
+		Record []struct {
+			Name string `xml:"name"`
+			Xref []struct {
+				Type string `xml:"type,attr"`
+				Data string `xml:"data,attr"`
+			} `xml:"xref"`
+			Desc struct {
+				Data string `xml:",innerxml"`
+			} `xml:"description,"`
+			MIB   string   `xml:"value"`
+			Alias []string `xml:"alias"`
+			MIME  string   `xml:"preferred_alias"`
+		} `xml:"record"`
+	} `xml:"registry"`
+}
+
+func main() {
+	r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml")
+	reg := &registry{}
+	if err := xml.NewDecoder(r).Decode(&reg); err != nil && err != io.EOF {
+		log.Fatalf("Error decoding charset registry: %v", err)
+	}
+	if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" {
+		log.Fatalf("Unexpected ID %s", reg.Registry[0].ID)
+	}
+
+	x := &indexInfo{}
+
+	for _, rec := range reg.Registry[0].Record {
+		mib := identifier.MIB(parseInt(rec.MIB))
+		x.addEntry(mib, rec.Name)
+		for _, a := range rec.Alias {
+			a = strings.Split(a, " ")[0] // strip comments.
+			x.addAlias(a, mib)
+			// MIB name aliases are prefixed with a "cs" (character set) in the
+			// registry to identify them as display names and to ensure that
+			// the name starts with a lowercase letter in case it is used as
+			// an identifier. We remove it to be left with a nice clean name.
+			if strings.HasPrefix(a, "cs") {
+				x.setName(2, a[2:])
+			}
+		}
+		if rec.MIME != "" {
+			x.addAlias(rec.MIME, mib)
+			x.setName(1, rec.MIME)
+		}
+	}
+
+	w := gen.NewCodeWriter()
+
+	fmt.Fprintln(w, `import "golang.org/x/text/encoding/internal/identifier"`)
+
+	writeIndex(w, x)
+
+	w.WriteGoFile("tables.go", "ianaindex")
+}
+
+type alias struct {
+	name string
+	mib  identifier.MIB
+}
+
+type indexInfo struct {
+	// compacted index from code to MIB
+	codeToMIB []identifier.MIB
+	alias     []alias
+	names     [][3]string
+}
+
+func (ii *indexInfo) Len() int {
+	return len(ii.codeToMIB)
+}
+
+func (ii *indexInfo) Less(a, b int) bool {
+	return ii.codeToMIB[a] < ii.codeToMIB[b]
+}
+
+func (ii *indexInfo) Swap(a, b int) {
+	ii.codeToMIB[a], ii.codeToMIB[b] = ii.codeToMIB[b], ii.codeToMIB[a]
+	// Co-sort the names.
+	ii.names[a], ii.names[b] = ii.names[b], ii.names[a]
+}
+
+func (ii *indexInfo) setName(i int, name string) {
+	ii.names[len(ii.names)-1][i] = name
+}
+
+func (ii *indexInfo) addEntry(mib identifier.MIB, name string) {
+	ii.names = append(ii.names, [3]string{name, name, name})
+	ii.addAlias(name, mib)
+	ii.codeToMIB = append(ii.codeToMIB, mib)
+}
+
+func (ii *indexInfo) addAlias(name string, mib identifier.MIB) {
+	// Don't add duplicates for the same mib. Adding duplicate aliases for
+	// different MIBs will cause the compiler to barf on an invalid map: great!.
+	for i := len(ii.alias) - 1; i >= 0 && ii.alias[i].mib == mib; i-- {
+		if ii.alias[i].name == name {
+			return
+		}
+	}
+	ii.alias = append(ii.alias, alias{name, mib})
+	lower := strings.ToLower(name)
+	if lower != name {
+		ii.addAlias(lower, mib)
+	}
+}
+
+const maxMIMENameLen = '0' - 1 // officially 40, but we leave some buffer.
+
+func writeIndex(w *gen.CodeWriter, x *indexInfo) {
+	sort.Stable(x)
+
+	// Write constants.
+	fmt.Fprintln(w, "const (")
+	for i, m := range x.codeToMIB {
+		if i == 0 {
+			fmt.Fprintf(w, "enc%d = iota\n", m)
+		} else {
+			fmt.Fprintf(w, "enc%d\n", m)
+		}
+	}
+	fmt.Fprintln(w, "numIANA")
+	fmt.Fprintln(w, ")")
+
+	w.WriteVar("ianaToMIB", x.codeToMIB)
+
+	var ianaNames, mibNames []string
+	for _, names := range x.names {
+		n := names[0]
+		if names[0] != names[1] {
+			// MIME names are mostly identical to IANA names. We share the
+			// tables by setting the first byte of the string to an index into
+			// the string itself (< maxMIMENameLen) to the IANA name. The MIME
+			// name immediately follows the index.
+			x := len(names[1]) + 1
+			if x > maxMIMENameLen {
+				log.Fatalf("MIME name length (%d) > %d", x, maxMIMENameLen)
+			}
+			n = string(x) + names[1] + names[0]
+		}
+		ianaNames = append(ianaNames, n)
+		mibNames = append(mibNames, names[2])
+	}
+
+	w.WriteVar("ianaNames", ianaNames)
+	w.WriteVar("mibNames", mibNames)
+
+	w.WriteComment(`
+	TODO: Instead of using a map, we could use binary search strings doing
+	on-the fly lower-casing per character. This allows to always avoid
+	allocation and will be considerably more compact.`)
+	fmt.Fprintln(w, "var ianaAliases = map[string]int{")
+	for _, a := range x.alias {
+		fmt.Fprintf(w, "%q: enc%d,\n", a.name, a.mib)
+	}
+	fmt.Fprintln(w, "}")
+}
+
+func parseInt(s string) int {
+	x, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		log.Fatalf("Could not parse integer: %v", err)
+	}
+	return int(x)
+}

vendor/golang.org/x/text/encoding/ianaindex/ianaindex.go

@@ -0,0 +1,209 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go
+
+// Package ianaindex maps names to Encodings as specified by the IANA registry.
+// This includes both the MIME and IANA names.
+//
+// See http://www.iana.org/assignments/character-sets/character-sets.xhtml for
+// more details.
+package ianaindex
+
+import (
+	"errors"
+	"sort"
+	"strings"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/encoding/traditionalchinese"
+	"golang.org/x/text/encoding/unicode"
+)
+
+// TODO: remove the "Status... incomplete" in the package doc comment.
+// TODO: allow users to specify their own aliases?
+// TODO: allow users to specify their own indexes?
+// TODO: allow canonicalizing names
+
+// NOTE: only use these top-level variables if we can get the linker to drop
+// the indexes when they are not used. Make them a function or perhaps only
+// support MIME otherwise.
+
+var (
+	// MIME is an index to map MIME names.
+	MIME *Index = mime
+
+	// IANA is an index that supports all names and aliases using IANA names as
+	// the canonical identifier.
+	IANA *Index = iana
+
+	// MIB is an index that associates the MIB display name with an Encoding.
+	MIB *Index = mib
+
+	mime = &Index{mimeName, ianaToMIB, ianaAliases, encodings[:]}
+	iana = &Index{ianaName, ianaToMIB, ianaAliases, encodings[:]}
+	mib  = &Index{mibName, ianaToMIB, ianaAliases, encodings[:]}
+)
+
+// Index maps names registered by IANA to Encodings.
+// Currently different Indexes only differ in the names they return for
+// encodings. In the future they may also differ in supported aliases.
+type Index struct {
+	names func(i int) string
+	toMIB []identifier.MIB // Sorted slice of supported MIBs
+	alias map[string]int
+	enc   []encoding.Encoding
+}
+
+var (
+	errInvalidName = errors.New("ianaindex: invalid encoding name")
+	errUnknown     = errors.New("ianaindex: unknown Encoding")
+	errUnsupported = errors.New("ianaindex: unsupported Encoding")
+)
+
+// Encoding returns an Encoding for IANA-registered names. Matching is
+// case-insensitive.
+func (x *Index) Encoding(name string) (encoding.Encoding, error) {
+	name = strings.TrimSpace(name)
+	// First try without lowercasing (possibly creating an allocation).
+	i, ok := x.alias[name]
+	if !ok {
+		i, ok = x.alias[strings.ToLower(name)]
+		if !ok {
+			return nil, errInvalidName
+		}
+	}
+	return x.enc[i], nil
+}
+
+// Name reports the canonical name of the given Encoding. It will return an
+// error if the e is not associated with a known encoding scheme.
+func (x *Index) Name(e encoding.Encoding) (string, error) {
+	id, ok := e.(identifier.Interface)
+	if !ok {
+		return "", errUnknown
+	}
+	mib, _ := id.ID()
+	if mib == 0 {
+		return "", errUnknown
+	}
+	v := findMIB(x.toMIB, mib)
+	if v == -1 {
+		return "", errUnsupported
+	}
+	return x.names(v), nil
+}
+
+// TODO: the coverage of this index is rather spotty. Allowing users to set
+// encodings would allow:
+// - users to increase coverage
+// - allow a partially loaded set of encodings in case the user doesn't need to
+//   them all.
+// - write an OS-specific wrapper for supported encodings and set them.
+// The exact definition of Set depends a bit on if and how we want to let users
+// write their own Encoding implementations. Also, it is not possible yet to
+// only partially load the encodings without doing some refactoring. Until this
+// is solved, we might as well not support Set.
+// // Set sets the e to be used for the encoding scheme identified by name. Only
+// // canonical names may be used. An empty name assigns e to its internally
+// // associated encoding scheme.
+// func (x *Index) Set(name string, e encoding.Encoding) error {
+// 	panic("TODO: implement")
+// }
+
+func findMIB(x []identifier.MIB, mib identifier.MIB) int {
+	i := sort.Search(len(x), func(i int) bool { return x[i] >= mib })
+	if i < len(x) && x[i] == mib {
+		return i
+	}
+	return -1
+}
+
+const maxMIMENameLen = '0' - 1 // officially 40, but we leave some buffer.
+
+func mimeName(x int) string {
+	n := ianaNames[x]
+	// See gen.go for a description of the encoding.
+	if n[0] <= maxMIMENameLen {
+		return n[1:n[0]]
+	}
+	return n
+}
+
+func ianaName(x int) string {
+	n := ianaNames[x]
+	// See gen.go for a description of the encoding.
+	if n[0] <= maxMIMENameLen {
+		return n[n[0]:]
+	}
+	return n
+}
+
+func mibName(x int) string {
+	return mibNames[x]
+}
+
+var encodings = [numIANA]encoding.Encoding{
+	enc106:  unicode.UTF8,
+	enc1015: unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
+	enc1013: unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
+	enc1014: unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
+	enc2028: charmap.CodePage037,
+	enc2011: charmap.CodePage437,
+	enc2009: charmap.CodePage850,
+	enc2010: charmap.CodePage852,
+	enc2046: charmap.CodePage855,
+	enc2089: charmap.CodePage858,
+	enc2048: charmap.CodePage860,
+	enc2013: charmap.CodePage862,
+	enc2050: charmap.CodePage863,
+	enc2052: charmap.CodePage865,
+	enc2086: charmap.CodePage866,
+	enc2102: charmap.CodePage1047,
+	enc2091: charmap.CodePage1140,
+	enc4:    charmap.ISO8859_1,
+	enc5:    charmap.ISO8859_2,
+	enc6:    charmap.ISO8859_3,
+	enc7:    charmap.ISO8859_4,
+	enc8:    charmap.ISO8859_5,
+	enc9:    charmap.ISO8859_6,
+	enc81:   charmap.ISO8859_6E,
+	enc82:   charmap.ISO8859_6I,
+	enc10:   charmap.ISO8859_7,
+	enc11:   charmap.ISO8859_8,
+	enc84:   charmap.ISO8859_8E,
+	enc85:   charmap.ISO8859_8I,
+	enc12:   charmap.ISO8859_9,
+	enc13:   charmap.ISO8859_10,
+	enc109:  charmap.ISO8859_13,
+	enc110:  charmap.ISO8859_14,
+	enc111:  charmap.ISO8859_15,
+	enc112:  charmap.ISO8859_16,
+	enc2084: charmap.KOI8R,
+	enc2088: charmap.KOI8U,
+	enc2027: charmap.Macintosh,
+	enc2109: charmap.Windows874,
+	enc2250: charmap.Windows1250,
+	enc2251: charmap.Windows1251,
+	enc2252: charmap.Windows1252,
+	enc2253: charmap.Windows1253,
+	enc2254: charmap.Windows1254,
+	enc2255: charmap.Windows1255,
+	enc2256: charmap.Windows1256,
+	enc2257: charmap.Windows1257,
+	enc2258: charmap.Windows1258,
+	enc18:   japanese.EUCJP,
+	enc39:   japanese.ISO2022JP,
+	enc17:   japanese.ShiftJIS,
+	enc38:   korean.EUCKR,
+	enc114:  simplifiedchinese.GB18030,
+	enc113:  simplifiedchinese.GBK,
+	enc2085: simplifiedchinese.HZGB2312,
+	enc2026: traditionalchinese.Big5,
+}

vendor/golang.org/x/text/encoding/ianaindex/ianaindex_test.go

@@ -0,0 +1,192 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ianaindex
+
+import (
+	"testing"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/encoding/traditionalchinese"
+	"golang.org/x/text/encoding/unicode"
+)
+
+var All = [][]encoding.Encoding{
+	unicode.All,
+	charmap.All,
+	japanese.All,
+	korean.All,
+	simplifiedchinese.All,
+	traditionalchinese.All,
+}
+
+// TestAllIANA tests whether an Encoding supported in x/text is defined by IANA but
+// not supported by this package.
+func TestAllIANA(t *testing.T) {
+	for _, ea := range All {
+		for _, e := range ea {
+			mib, _ := e.(identifier.Interface).ID()
+			if x := findMIB(ianaToMIB, mib); x != -1 && encodings[x] == nil {
+				t.Errorf("supported MIB %v (%v) not in index", mib, e)
+			}
+		}
+	}
+}
+
+// TestNotSupported reports the encodings in IANA, but not by x/text.
+func TestNotSupported(t *testing.T) {
+	mibs := map[identifier.MIB]bool{}
+	for _, ea := range All {
+		for _, e := range ea {
+			mib, _ := e.(identifier.Interface).ID()
+			mibs[mib] = true
+		}
+	}
+
+	// Many encodings in the IANA index will likely not be suppored by the
+	// Go encodings. That is fine.
+	// TODO: consider wheter we should add this test.
+	// for code, mib := range ianaToMIB {
+	// 	t.Run(fmt.Sprint("IANA:", mib), func(t *testing.T) {
+	// 		if !mibs[mib] {
+	// 			t.Skipf("IANA encoding %s (MIB %v) not supported",
+	// 				ianaNames[code], mib)
+	// 		}
+	// 	})
+	// }
+}
+
+func TestEncoding(t *testing.T) {
+	testCases := []struct {
+		index     *Index
+		name      string
+		canonical string
+		err       error
+	}{
+		{MIME, "utf-8", "UTF-8", nil},
+		{MIME, "  utf-8  ", "UTF-8", nil},
+		{MIME, "  l5  ", "ISO-8859-9", nil},
+		{MIME, "latin5 ", "ISO-8859-9", nil},
+		{MIME, "LATIN5 ", "ISO-8859-9", nil},
+		{MIME, "latin 5", "", errInvalidName},
+		{MIME, "latin-5", "", errInvalidName},
+
+		{IANA, "utf-8", "UTF-8", nil},
+		{IANA, "  utf-8  ", "UTF-8", nil},
+		{IANA, "  l5  ", "ISO_8859-9:1989", nil},
+		{IANA, "latin5 ", "ISO_8859-9:1989", nil},
+		{IANA, "LATIN5 ", "ISO_8859-9:1989", nil},
+		{IANA, "latin 5", "", errInvalidName},
+		{IANA, "latin-5", "", errInvalidName},
+
+		{MIB, "utf-8", "UTF8", nil},
+		{MIB, "  utf-8  ", "UTF8", nil},
+		{MIB, "  l5  ", "ISOLatin5", nil},
+		{MIB, "latin5 ", "ISOLatin5", nil},
+		{MIB, "LATIN5 ", "ISOLatin5", nil},
+		{MIB, "latin 5", "", errInvalidName},
+		{MIB, "latin-5", "", errInvalidName},
+	}
+	for i, tc := range testCases {
+		enc, err := tc.index.Encoding(tc.name)
+		if err != tc.err {
+			t.Errorf("%d: error was %v; want %v", i, err, tc.err)
+		}
+		if err != nil {
+			continue
+		}
+		if got, err := tc.index.Name(enc); got != tc.canonical {
+			t.Errorf("%d: Name(Encoding(%q)) = %q; want %q (%v)", i, tc.name, got, tc.canonical, err)
+		}
+	}
+}
+
+func TestTables(t *testing.T) {
+	for i, x := range []*Index{MIME, IANA} {
+		for name, index := range x.alias {
+			got, err := x.Encoding(name)
+			if err != nil {
+				t.Errorf("%d%s:err: unexpected error %v", i, name, err)
+			}
+			if want := x.enc[index]; got != want {
+				t.Errorf("%d%s:encoding: got %v; want %v", i, name, got, want)
+			}
+			if got != nil {
+				mib, _ := got.(identifier.Interface).ID()
+				if i := findMIB(x.toMIB, mib); i != index {
+					t.Errorf("%d%s:mib: got %d; want %d", i, name, i, index)
+				}
+			}
+		}
+	}
+}
+
+type unsupported struct {
+	encoding.Encoding
+}
+
+func (unsupported) ID() (identifier.MIB, string) { return 9999, "" }
+
+func TestName(t *testing.T) {
+	testCases := []struct {
+		desc string
+		enc  encoding.Encoding
+		f    func(e encoding.Encoding) (string, error)
+		name string
+		err  error
+	}{{
+		"defined encoding",
+		charmap.ISO8859_2,
+		MIME.Name,
+		"ISO-8859-2",
+		nil,
+	}, {
+		"defined Unicode encoding",
+		unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
+		IANA.Name,
+		"UTF-16BE",
+		nil,
+	}, {
+		"another defined Unicode encoding",
+		unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
+		MIME.Name,
+		"UTF-16",
+		nil,
+	}, {
+		"unknown Unicode encoding",
+		unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM),
+		MIME.Name,
+		"",
+		errUnknown,
+	}, {
+		"undefined encoding",
+		unsupported{},
+		MIME.Name,
+		"",
+		errUnsupported,
+	}, {
+		"undefined other encoding in HTML standard",
+		charmap.CodePage437,
+		IANA.Name,
+		"IBM437",
+		nil,
+	}, {
+		"unknown encoding",
+		encoding.Nop,
+		IANA.Name,
+		"",
+		errUnknown,
+	}}
+	for i, tc := range testCases {
+		name, err := tc.f(tc.enc)
+		if name != tc.name || err != tc.err {
+			t.Errorf("%d:%s: got %q, %v; want %q, %v", i, tc.desc, name, err, tc.name, tc.err)
+		}
+	}
+}