Checking in vendor folder for ease of using go get.

vendor/golang.org/x/text/encoding/traditionalchinese/all_test.go

@@ -0,0 +1,114 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package traditionalchinese
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal"
+	"golang.org/x/text/encoding/internal/enctest"
+	"golang.org/x/text/transform"
+)
+
+func dec(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
+	return "Decode", e.NewDecoder(), nil
+}
+func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
+	return "Encode", e.NewEncoder(), internal.ErrASCIIReplacement
+}
+
+func TestNonRepertoire(t *testing.T) {
+	testCases := []struct {
+		init      func(e encoding.Encoding) (string, transform.Transformer, error)
+		e         encoding.Encoding
+		src, want string
+	}{
+		{dec, Big5, "\x80", "\ufffd"},
+		{dec, Big5, "\x81", "\ufffd"},
+		{dec, Big5, "\x81\x30", "\ufffd\x30"},
+		{dec, Big5, "\x81\x40", "\ufffd"},
+		{dec, Big5, "\x81\xa0", "\ufffd"},
+		{dec, Big5, "\xff", "\ufffd"},
+
+		{enc, Big5, "갂", ""},
+		{enc, Big5, "a갂", "a"},
+		{enc, Big5, "\u43f0갂", "\x87@"},
+	}
+	for _, tc := range testCases {
+		dir, tr, wantErr := tc.init(tc.e)
+		t.Run(fmt.Sprintf("%s/%v/%q", dir, tc.e, tc.src), func(t *testing.T) {
+			dst := make([]byte, 100)
+			src := []byte(tc.src)
+			for i := 0; i <= len(tc.src); i++ {
+				nDst, nSrc, err := tr.Transform(dst, src[:i], false)
+				if err != nil && err != transform.ErrShortSrc && err != wantErr {
+					t.Fatalf("error on first call to Transform: %v", err)
+				}
+				n, _, err := tr.Transform(dst[nDst:], src[nSrc:], true)
+				nDst += n
+				if err != wantErr {
+					t.Fatalf("(%q|%q): got %v; want %v", tc.src[:i], tc.src[i:], err, wantErr)
+				}
+				if got := string(dst[:nDst]); got != tc.want {
+					t.Errorf("(%q|%q):\ngot  %q\nwant %q", tc.src[:i], tc.src[i:], got, tc.want)
+				}
+			}
+		})
+	}
+}
+
+func TestBasics(t *testing.T) {
+	// The encoded forms can be verified by the iconv program:
+	// $ echo 月日は百代 | iconv -f UTF-8 -t SHIFT-JIS | xxd
+	testCases := []struct {
+		e         encoding.Encoding
+		encPrefix string
+		encSuffix string
+		encoded   string
+		utf8      string
+	}{{
+		e:       Big5,
+		encoded: "A\x87\x40\x87\x41\x87\x45\xa1\x40\xfe\xfd\xfe\xfeZ\xa3\xe1",
+		utf8:    "A\u43f0\u4c32\U00027267\u3000\U0002910d\u79d4Z€",
+	}, {
+		e: Big5,
+		encoded: "\xaa\xe1\xb6\xa1\xa4\x40\xb3\xfd\xb0\x73\xa1\x41\xbf\x57\xb0\x75" +
+			"\xb5\x4c\xac\xdb\xbf\xcb\xa1\x43",
+		utf8: "花間一壺酒，獨酌無相親。",
+	}}
+
+	for _, tc := range testCases {
+		enctest.TestEncoding(t, tc.e, tc.encoded, tc.utf8, "", "")
+	}
+}
+
+func TestFiles(t *testing.T) { enctest.TestFile(t, Big5) }
+
+func BenchmarkEncoding(b *testing.B) { enctest.Benchmark(b, Big5) }
+
+// TestBig5CircumflexAndMacron tests the special cases listed in
+// http://encoding.spec.whatwg.org/#big5
+// Note that these special cases aren't preserved by round-tripping through
+// decoding and encoding (since
+// http://encoding.spec.whatwg.org/index-big5.txt does not have an entry for
+// U+0304 or U+030C), so we can't test this in TestBasics.
+func TestBig5CircumflexAndMacron(t *testing.T) {
+	src := "\x88\x5f\x88\x60\x88\x61\x88\x62\x88\x63\x88\x64\x88\x65\x88\x66 " +
+		"\x88\xa2\x88\xa3\x88\xa4\x88\xa5\x88\xa6"
+	want := "ÓǑÒ\u00ca\u0304Ế\u00ca\u030cỀÊ " +
+		"ü\u00ea\u0304ế\u00ea\u030cề"
+	dst, err := ioutil.ReadAll(transform.NewReader(
+		strings.NewReader(src), Big5.NewDecoder()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(dst); got != want {
+		t.Fatalf("\ngot  %q\nwant %q", got, want)
+	}
+}

vendor/golang.org/x/text/encoding/traditionalchinese/big5.go

@@ -0,0 +1,199 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package traditionalchinese
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+// All is a list of all defined encodings in this package.
+var All = []encoding.Encoding{Big5}
+
+// Big5 is the Big5 encoding, also known as Code Page 950.
+var Big5 encoding.Encoding = &big5
+
+var big5 = internal.Encoding{
+	&internal.SimpleEncoding{big5Decoder{}, big5Encoder{}},
+	"Big5",
+	identifier.Big5,
+}
+
+type big5Decoder struct{ transform.NopResetter }
+
+func (big5Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size, s := rune(0), 0, ""
+loop:
+	for ; nSrc < len(src); nSrc += size {
+		switch c0 := src[nSrc]; {
+		case c0 < utf8.RuneSelf:
+			r, size = rune(c0), 1
+
+		case 0x81 <= c0 && c0 < 0xff:
+			if nSrc+1 >= len(src) {
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				goto write
+			}
+			c1 := src[nSrc+1]
+			switch {
+			case 0x40 <= c1 && c1 < 0x7f:
+				c1 -= 0x40
+			case 0xa1 <= c1 && c1 < 0xff:
+				c1 -= 0x62
+			case c1 < 0x40:
+				r, size = utf8.RuneError, 1
+				goto write
+			default:
+				r, size = utf8.RuneError, 2
+				goto write
+			}
+			r, size = '\ufffd', 2
+			if i := int(c0-0x81)*157 + int(c1); i < len(decode) {
+				if 1133 <= i && i < 1167 {
+					// The two-rune special cases for LATIN CAPITAL / SMALL E WITH CIRCUMFLEX
+					// AND MACRON / CARON are from http://encoding.spec.whatwg.org/#big5
+					switch i {
+					case 1133:
+						s = "\u00CA\u0304"
+						goto writeStr
+					case 1135:
+						s = "\u00CA\u030C"
+						goto writeStr
+					case 1164:
+						s = "\u00EA\u0304"
+						goto writeStr
+					case 1166:
+						s = "\u00EA\u030C"
+						goto writeStr
+					}
+				}
+				r = rune(decode[i])
+				if r == 0 {
+					r = '\ufffd'
+				}
+			}
+
+		default:
+			r, size = utf8.RuneError, 1
+		}
+
+	write:
+		if nDst+utf8.RuneLen(r) > len(dst) {
+			err = transform.ErrShortDst
+			break loop
+		}
+		nDst += utf8.EncodeRune(dst[nDst:], r)
+		continue loop
+
+	writeStr:
+		if nDst+len(s) > len(dst) {
+			err = transform.ErrShortDst
+			break loop
+		}
+		nDst += copy(dst[nDst:], s)
+		continue loop
+	}
+	return nDst, nSrc, err
+}
+
+type big5Encoder struct{ transform.NopResetter }
+
+func (big5Encoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size := rune(0), 0
+	for ; nSrc < len(src); nSrc += size {
+		r = rune(src[nSrc])
+
+		// Decode a 1-byte rune.
+		if r < utf8.RuneSelf {
+			size = 1
+			if nDst >= len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			dst[nDst] = uint8(r)
+			nDst++
+			continue
+
+		} else {
+			// Decode a multi-byte rune.
+			r, size = utf8.DecodeRune(src[nSrc:])
+			if size == 1 {
+				// All valid runes of size 1 (those below utf8.RuneSelf) were
+				// handled above. We have invalid UTF-8 or we haven't seen the
+				// full character yet.
+				if !atEOF && !utf8.FullRune(src[nSrc:]) {
+					err = transform.ErrShortSrc
+					break
+				}
+			}
+		}
+
+		if r >= utf8.RuneSelf {
+			// func init checks that the switch covers all tables.
+			switch {
+			case encode0Low <= r && r < encode0High:
+				if r = rune(encode0[r-encode0Low]); r != 0 {
+					goto write2
+				}
+			case encode1Low <= r && r < encode1High:
+				if r = rune(encode1[r-encode1Low]); r != 0 {
+					goto write2
+				}
+			case encode2Low <= r && r < encode2High:
+				if r = rune(encode2[r-encode2Low]); r != 0 {
+					goto write2
+				}
+			case encode3Low <= r && r < encode3High:
+				if r = rune(encode3[r-encode3Low]); r != 0 {
+					goto write2
+				}
+			case encode4Low <= r && r < encode4High:
+				if r = rune(encode4[r-encode4Low]); r != 0 {
+					goto write2
+				}
+			case encode5Low <= r && r < encode5High:
+				if r = rune(encode5[r-encode5Low]); r != 0 {
+					goto write2
+				}
+			case encode6Low <= r && r < encode6High:
+				if r = rune(encode6[r-encode6Low]); r != 0 {
+					goto write2
+				}
+			case encode7Low <= r && r < encode7High:
+				if r = rune(encode7[r-encode7Low]); r != 0 {
+					goto write2
+				}
+			}
+			err = internal.ErrASCIIReplacement
+			break
+		}
+
+	write2:
+		if nDst+2 > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst+0] = uint8(r >> 8)
+		dst[nDst+1] = uint8(r)
+		nDst += 2
+		continue
+	}
+	return nDst, nSrc, err
+}
+
+func init() {
+	// Check that the hard-coded encode switch covers all tables.
+	if numEncodeTables != 8 {
+		panic("bad numEncodeTables")
+	}
+}

vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go

@@ -0,0 +1,140 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This program generates tables.go:
+//	go run maketables.go | gofmt > tables.go
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+)
+
+func main() {
+	fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
+	fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n")
+	fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n")
+
+	res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt")
+	if err != nil {
+		log.Fatalf("Get: %v", err)
+	}
+	defer res.Body.Close()
+
+	mapping := [65536]uint32{}
+	reverse := [65536 * 4]uint16{}
+
+	scanner := bufio.NewScanner(res.Body)
+	for scanner.Scan() {
+		s := strings.TrimSpace(scanner.Text())
+		if s == "" || s[0] == '#' {
+			continue
+		}
+		x, y := uint16(0), uint32(0)
+		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
+			log.Fatalf("could not parse %q", s)
+		}
+		if x < 0 || 126*157 <= x {
+			log.Fatalf("Big5 code %d is out of range", x)
+		}
+		mapping[x] = y
+
+		// The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that
+		// "The index pointer for code point in index is the first pointer
+		// corresponding to code point in index", which would normally mean
+		// that the code below should be guarded by "if reverse[y] == 0", but
+		// last instead of first seems to match the behavior of
+		// "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in
+		// http://encoding.spec.whatwg.org/index-big5.txt, as index 2148
+		// (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc")
+		// and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc".
+		c0, c1 := x/157, x%157
+		if c1 < 0x3f {
+			c1 += 0x40
+		} else {
+			c1 += 0x62
+		}
+		reverse[y] = (0x81+c0)<<8 | c1
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatalf("scanner error: %v", err)
+	}
+
+	fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n")
+	fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n")
+	fmt.Printf("var decode = [...]uint32{\n")
+	for i, v := range mapping {
+		if v != 0 {
+			fmt.Printf("\t%d: 0x%08X,\n", i, v)
+		}
+	}
+	fmt.Printf("}\n\n")
+
+	// Any run of at least separation continuous zero entries in the reverse map will
+	// be a separate encode table.
+	const separation = 1024
+
+	intervals := []interval(nil)
+	low, high := -1, -1
+	for i, v := range reverse {
+		if v == 0 {
+			continue
+		}
+		if low < 0 {
+			low = i
+		} else if i-high >= separation {
+			if high >= 0 {
+				intervals = append(intervals, interval{low, high})
+			}
+			low = i
+		}
+		high = i + 1
+	}
+	if high >= 0 {
+		intervals = append(intervals, interval{low, high})
+	}
+	sort.Sort(byDecreasingLength(intervals))
+
+	fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
+	fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n")
+	fmt.Printf("// sorted by decreasing length.\n")
+	for i, v := range intervals {
+		fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high)
+	}
+	fmt.Printf("\n")
+
+	for i, v := range intervals {
+		fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
+		fmt.Printf("var encode%d = [...]uint16{\n", i)
+		for j := v.low; j < v.high; j++ {
+			x := reverse[j]
+			if x == 0 {
+				continue
+			}
+			fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
+		}
+		fmt.Printf("}\n\n")
+	}
+}
+
+// interval is a half-open interval [low, high).
+type interval struct {
+	low, high int
+}
+
+func (i interval) len() int { return i.high - i.low }
+
+// byDecreasingLength sorts intervals by decreasing length.
+type byDecreasingLength []interval
+
+func (b byDecreasingLength) Len() int           { return len(b) }
+func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
+func (b byDecreasingLength) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }