Checking in vendor folder for ease of using go get.

vendor/golang.org/x/text/encoding/internal/enctest/enctest.go

@@ -0,0 +1,180 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package enctest
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+// Encoder or Decoder
+type Transcoder interface {
+	transform.Transformer
+	Bytes([]byte) ([]byte, error)
+	String(string) (string, error)
+}
+
+func TestEncoding(t *testing.T, e encoding.Encoding, encoded, utf8, prefix, suffix string) {
+	for _, direction := range []string{"Decode", "Encode"} {
+		t.Run(fmt.Sprintf("%v/%s", e, direction), func(t *testing.T) {
+
+			var coder Transcoder
+			var want, src, wPrefix, sPrefix, wSuffix, sSuffix string
+			if direction == "Decode" {
+				coder, want, src = e.NewDecoder(), utf8, encoded
+				wPrefix, sPrefix, wSuffix, sSuffix = "", prefix, "", suffix
+			} else {
+				coder, want, src = e.NewEncoder(), encoded, utf8
+				wPrefix, sPrefix, wSuffix, sSuffix = prefix, "", suffix, ""
+			}
+
+			dst := make([]byte, len(wPrefix)+len(want)+len(wSuffix))
+			nDst, nSrc, err := coder.Transform(dst, []byte(sPrefix+src+sSuffix), true)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if nDst != len(wPrefix)+len(want)+len(wSuffix) {
+				t.Fatalf("nDst got %d, want %d",
+					nDst, len(wPrefix)+len(want)+len(wSuffix))
+			}
+			if nSrc != len(sPrefix)+len(src)+len(sSuffix) {
+				t.Fatalf("nSrc got %d, want %d",
+					nSrc, len(sPrefix)+len(src)+len(sSuffix))
+			}
+			if got := string(dst); got != wPrefix+want+wSuffix {
+				t.Fatalf("\ngot  %q\nwant %q", got, wPrefix+want+wSuffix)
+			}
+
+			for _, n := range []int{0, 1, 2, 10, 123, 4567} {
+				input := sPrefix + strings.Repeat(src, n) + sSuffix
+				g, err := coder.String(input)
+				if err != nil {
+					t.Fatalf("Bytes: n=%d: %v", n, err)
+				}
+				if len(g) == 0 && len(input) == 0 {
+					// If the input is empty then the output can be empty,
+					// regardless of whatever wPrefix is.
+					continue
+				}
+				got1, want1 := string(g), wPrefix+strings.Repeat(want, n)+wSuffix
+				if got1 != want1 {
+					t.Fatalf("ReadAll: n=%d\ngot  %q\nwant %q",
+						n, trim(got1), trim(want1))
+				}
+			}
+		})
+	}
+}
+
+func TestFile(t *testing.T, e encoding.Encoding) {
+	for _, dir := range []string{"Decode", "Encode"} {
+		t.Run(fmt.Sprintf("%s/%s", e, dir), func(t *testing.T) {
+			dst, src, transformer, err := load(dir, e)
+			if err != nil {
+				t.Fatalf("load: %v", err)
+			}
+			buf, err := transformer.Bytes(src)
+			if err != nil {
+				t.Fatalf("transform: %v", err)
+			}
+			if !bytes.Equal(buf, dst) {
+				t.Error("transformed bytes did not match golden file")
+			}
+		})
+	}
+}
+
+func Benchmark(b *testing.B, enc encoding.Encoding) {
+	for _, direction := range []string{"Decode", "Encode"} {
+		b.Run(fmt.Sprintf("%s/%s", enc, direction), func(b *testing.B) {
+			_, src, transformer, err := load(direction, enc)
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.SetBytes(int64(len(src)))
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				r := transform.NewReader(bytes.NewReader(src), transformer)
+				io.Copy(ioutil.Discard, r)
+			}
+		})
+	}
+}
+
+// testdataFiles are files in testdata/*.txt.
+var testdataFiles = []struct {
+	mib           identifier.MIB
+	basename, ext string
+}{
+	{identifier.Windows1252, "candide", "windows-1252"},
+	{identifier.EUCPkdFmtJapanese, "rashomon", "euc-jp"},
+	{identifier.ISO2022JP, "rashomon", "iso-2022-jp"},
+	{identifier.ShiftJIS, "rashomon", "shift-jis"},
+	{identifier.EUCKR, "unsu-joh-eun-nal", "euc-kr"},
+	{identifier.GBK, "sunzi-bingfa-simplified", "gbk"},
+	{identifier.HZGB2312, "sunzi-bingfa-gb-levels-1-and-2", "hz-gb2312"},
+	{identifier.Big5, "sunzi-bingfa-traditional", "big5"},
+	{identifier.UTF16LE, "candide", "utf-16le"},
+	{identifier.UTF8, "candide", "utf-8"},
+	{identifier.UTF32BE, "candide", "utf-32be"},
+
+	// GB18030 is a superset of GBK and is nominally a Simplified Chinese
+	// encoding, but it can also represent the entire Basic Multilingual
+	// Plane, including codepoints like 'â' that aren't encodable by GBK.
+	// GB18030 on Simplified Chinese should perform similarly to GBK on
+	// Simplified Chinese. GB18030 on "candide" is more interesting.
+	{identifier.GB18030, "candide", "gb18030"},
+}
+
+func load(direction string, enc encoding.Encoding) ([]byte, []byte, Transcoder, error) {
+	basename, ext, count := "", "", 0
+	for _, tf := range testdataFiles {
+		if mib, _ := enc.(identifier.Interface).ID(); tf.mib == mib {
+			basename, ext = tf.basename, tf.ext
+			count++
+		}
+	}
+	if count != 1 {
+		if count == 0 {
+			return nil, nil, nil, fmt.Errorf("no testdataFiles for %s", enc)
+		}
+		return nil, nil, nil, fmt.Errorf("too many testdataFiles for %s", enc)
+	}
+	dstFile := fmt.Sprintf("../testdata/%s-%s.txt", basename, ext)
+	srcFile := fmt.Sprintf("../testdata/%s-utf-8.txt", basename)
+	var coder Transcoder = encoding.ReplaceUnsupported(enc.NewEncoder())
+	if direction == "Decode" {
+		dstFile, srcFile = srcFile, dstFile
+		coder = enc.NewDecoder()
+	}
+	dst, err := ioutil.ReadFile(dstFile)
+	if err != nil {
+		if dst, err = ioutil.ReadFile("../" + dstFile); err != nil {
+			return nil, nil, nil, err
+		}
+	}
+	src, err := ioutil.ReadFile(srcFile)
+	if err != nil {
+		if src, err = ioutil.ReadFile("../" + srcFile); err != nil {
+			return nil, nil, nil, err
+		}
+	}
+	return dst, src, coder, nil
+}
+
+func trim(s string) string {
+	if len(s) < 120 {
+		return s
+	}
+	return s[:50] + "..." + s[len(s)-50:]
+}

vendor/golang.org/x/text/encoding/internal/identifier/gen.go

@@ -0,0 +1,137 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+)
+
+type registry struct {
+	XMLName  xml.Name `xml:"registry"`
+	Updated  string   `xml:"updated"`
+	Registry []struct {
+		ID     string `xml:"id,attr"`
+		Record []struct {
+			Name string `xml:"name"`
+			Xref []struct {
+				Type string `xml:"type,attr"`
+				Data string `xml:"data,attr"`
+			} `xml:"xref"`
+			Desc struct {
+				Data string `xml:",innerxml"`
+				// Any []struct {
+				// 	Data string `xml:",chardata"`
+				// } `xml:",any"`
+				// Data string `xml:",chardata"`
+			} `xml:"description,"`
+			MIB   string   `xml:"value"`
+			Alias []string `xml:"alias"`
+			MIME  string   `xml:"preferred_alias"`
+		} `xml:"record"`
+	} `xml:"registry"`
+}
+
+func main() {
+	r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml")
+	reg := &registry{}
+	if err := xml.NewDecoder(r).Decode(&reg); err != nil && err != io.EOF {
+		log.Fatalf("Error decoding charset registry: %v", err)
+	}
+	if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" {
+		log.Fatalf("Unexpected ID %s", reg.Registry[0].ID)
+	}
+
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, "const (\n")
+	for _, rec := range reg.Registry[0].Record {
+		constName := ""
+		for _, a := range rec.Alias {
+			if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 {
+				// Some of the constant definitions have comments in them. Strip those.
+				constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0])
+			}
+		}
+		if constName == "" {
+			switch rec.MIB {
+			case "2085":
+				constName = "HZGB2312" // Not listed as alias for some reason.
+			default:
+				log.Fatalf("No cs alias defined for %s.", rec.MIB)
+			}
+		}
+		if rec.MIME != "" {
+			rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME)
+		}
+		fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME)
+		if len(rec.Desc.Data) > 0 {
+			fmt.Fprint(w, "// ")
+			d := xml.NewDecoder(strings.NewReader(rec.Desc.Data))
+			inElem := true
+			attr := ""
+			for {
+				t, err := d.Token()
+				if err != nil {
+					if err != io.EOF {
+						log.Fatal(err)
+					}
+					break
+				}
+				switch x := t.(type) {
+				case xml.CharData:
+					attr = "" // Don't need attribute info.
+					a := bytes.Split([]byte(x), []byte("\n"))
+					for i, b := range a {
+						if b = bytes.TrimSpace(b); len(b) != 0 {
+							if !inElem && i > 0 {
+								fmt.Fprint(w, "\n// ")
+							}
+							inElem = false
+							fmt.Fprintf(w, "%s ", string(b))
+						}
+					}
+				case xml.StartElement:
+					if x.Name.Local == "xref" {
+						inElem = true
+						use := false
+						for _, a := range x.Attr {
+							if a.Name.Local == "type" {
+								use = use || a.Value != "person"
+							}
+							if a.Name.Local == "data" && use {
+								attr = a.Value + " "
+							}
+						}
+					}
+				case xml.EndElement:
+					inElem = false
+					fmt.Fprint(w, attr)
+				}
+			}
+			fmt.Fprint(w, "\n")
+		}
+		for _, x := range rec.Xref {
+			switch x.Type {
+			case "rfc":
+				fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data))
+			case "uri":
+				fmt.Fprintf(w, "// Reference: %s\n", x.Data)
+			}
+		}
+		fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB)
+		fmt.Fprintln(w)
+	}
+	fmt.Fprintln(w, ")")
+
+	gen.WriteGoFile("mib.go", "identifier", w.Bytes())
+}

vendor/golang.org/x/text/encoding/internal/identifier/identifier.go

@@ -0,0 +1,81 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go
+
+// Package identifier defines the contract between implementations of Encoding
+// and Index by defining identifiers that uniquely identify standardized coded
+// character sets (CCS) and character encoding schemes (CES), which we will
+// together refer to as encodings, for which Encoding implementations provide
+// converters to and from UTF-8. This package is typically only of concern to
+// implementers of Indexes and Encodings.
+//
+// One part of the identifier is the MIB code, which is defined by IANA and
+// uniquely identifies a CCS or CES. Each code is associated with data that
+// references authorities, official documentation as well as aliases and MIME
+// names.
+//
+// Not all CESs are covered by the IANA registry. The "other" string that is
+// returned by ID can be used to identify other character sets or versions of
+// existing ones.
+//
+// It is recommended that each package that provides a set of Encodings provide
+// the All and Common variables to reference all supported encodings and
+// commonly used subset. This allows Index implementations to include all
+// available encodings without explicitly referencing or knowing about them.
+package identifier
+
+// Note: this package is internal, but could be made public if there is a need
+// for writing third-party Indexes and Encodings.
+
+// References:
+// - http://source.icu-project.org/repos/icu/icu/trunk/source/data/mappings/convrtrs.txt
+// - http://www.iana.org/assignments/character-sets/character-sets.xhtml
+// - http://www.iana.org/assignments/ianacharset-mib/ianacharset-mib
+// - http://www.ietf.org/rfc/rfc2978.txt
+// - http://www.unicode.org/reports/tr22/
+// - http://www.w3.org/TR/encoding/
+// - https://encoding.spec.whatwg.org/
+// - https://encoding.spec.whatwg.org/encodings.json
+// - https://tools.ietf.org/html/rfc6657#section-5
+
+// Interface can be implemented by Encodings to define the CCS or CES for which
+// it implements conversions.
+type Interface interface {
+	// ID returns an encoding identifier. Exactly one of the mib and other
+	// values should be non-zero.
+	//
+	// In the usual case it is only necessary to indicate the MIB code. The
+	// other string can be used to specify encodings for which there is no MIB,
+	// such as "x-mac-dingbat".
+	//
+	// The other string may only contain the characters a-z, A-Z, 0-9, - and _.
+	ID() (mib MIB, other string)
+
+	// NOTE: the restrictions on the encoding are to allow extending the syntax
+	// with additional information such as versions, vendors and other variants.
+}
+
+// A MIB identifies an encoding. It is derived from the IANA MIB codes and adds
+// some identifiers for some encodings that are not covered by the IANA
+// standard.
+//
+// See http://www.iana.org/assignments/ianacharset-mib.
+type MIB uint16
+
+// These additional MIB types are not defined in IANA. They are added because
+// they are common and defined within the text repo.
+const (
+	// Unofficial marks the start of encodings not registered by IANA.
+	Unofficial MIB = 10000 + iota
+
+	// Replacement is the WhatWG replacement encoding.
+	Replacement
+
+	// XUserDefined is the code for x-user-defined.
+	XUserDefined
+
+	// MacintoshCyrillic is the code for x-mac-cyrillic.
+	MacintoshCyrillic
+)

vendor/golang.org/x/text/encoding/internal/internal.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package internal contains code that is shared among encoding implementations.
+package internal
+
+import (
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+// Encoding is an implementation of the Encoding interface that adds the String
+// and ID methods to an existing encoding.
+type Encoding struct {
+	encoding.Encoding
+	Name string
+	MIB  identifier.MIB
+}
+
+// _ verifies that Encoding implements identifier.Interface.
+var _ identifier.Interface = (*Encoding)(nil)
+
+func (e *Encoding) String() string {
+	return e.Name
+}
+
+func (e *Encoding) ID() (mib identifier.MIB, other string) {
+	return e.MIB, ""
+}
+
+// SimpleEncoding is an Encoding that combines two Transformers.
+type SimpleEncoding struct {
+	Decoder transform.Transformer
+	Encoder transform.Transformer
+}
+
+func (e *SimpleEncoding) NewDecoder() *encoding.Decoder {
+	return &encoding.Decoder{Transformer: e.Decoder}
+}
+
+func (e *SimpleEncoding) NewEncoder() *encoding.Encoder {
+	return &encoding.Encoder{Transformer: e.Encoder}
+}
+
+// FuncEncoding is an Encoding that combines two functions returning a new
+// Transformer.
+type FuncEncoding struct {
+	Decoder func() transform.Transformer
+	Encoder func() transform.Transformer
+}
+
+func (e FuncEncoding) NewDecoder() *encoding.Decoder {
+	return &encoding.Decoder{Transformer: e.Decoder()}
+}
+
+func (e FuncEncoding) NewEncoder() *encoding.Encoder {
+	return &encoding.Encoder{Transformer: e.Encoder()}
+}
+
+// A RepertoireError indicates a rune is not in the repertoire of a destination
+// encoding. It is associated with an encoding-specific suggested replacement
+// byte.
+type RepertoireError byte
+
+// Error implements the error interrface.
+func (r RepertoireError) Error() string {
+	return "encoding: rune not supported by encoding."
+}
+
+// Replacement returns the replacement string associated with this error.
+func (r RepertoireError) Replacement() byte { return byte(r) }
+
+var ErrASCIIReplacement = RepertoireError(encoding.ASCIISub)