Switch to using the dep tool and update all the dependencies

vendor/golang.org/x/text/collate/collate.go

@@ -0,0 +1,403 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO: remove hard-coded versions when we have implemented fractional weights.
+// The current implementation is incompatible with later CLDR versions.
+//go:generate go run maketables.go -cldr=23 -unicode=6.2.0
+
+// Package collate contains types for comparing and sorting Unicode strings
+// according to a given collation order.
+package collate // import "golang.org/x/text/collate"
+
+import (
+	"bytes"
+	"strings"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+)
+
+// Collator provides functionality for comparing strings for a given
+// collation order.
+type Collator struct {
+	options
+
+	sorter sorter
+
+	_iter [2]iter
+}
+
+func (c *Collator) iter(i int) *iter {
+	// TODO: evaluate performance for making the second iterator optional.
+	return &c._iter[i]
+}
+
+// Supported returns the list of languages for which collating differs from its parent.
+func Supported() []language.Tag {
+	// TODO: use language.Coverage instead.
+
+	t := make([]language.Tag, len(tags))
+	copy(t, tags)
+	return t
+}
+
+func init() {
+	ids := strings.Split(availableLocales, ",")
+	tags = make([]language.Tag, len(ids))
+	for i, s := range ids {
+		tags[i] = language.Raw.MustParse(s)
+	}
+}
+
+var tags []language.Tag
+
+// New returns a new Collator initialized for the given locale.
+func New(t language.Tag, o ...Option) *Collator {
+	index := colltab.MatchLang(t, tags)
+	c := newCollator(getTable(locales[index]))
+
+	// Set options from the user-supplied tag.
+	c.setFromTag(t)
+
+	// Set the user-supplied options.
+	c.setOptions(o)
+
+	c.init()
+	return c
+}
+
+// NewFromTable returns a new Collator for the given Weighter.
+func NewFromTable(w colltab.Weighter, o ...Option) *Collator {
+	c := newCollator(w)
+	c.setOptions(o)
+	c.init()
+	return c
+}
+
+func (c *Collator) init() {
+	if c.numeric {
+		c.t = colltab.NewNumericWeighter(c.t)
+	}
+	c._iter[0].init(c)
+	c._iter[1].init(c)
+}
+
+// Buffer holds keys generated by Key and KeyString.
+type Buffer struct {
+	buf [4096]byte
+	key []byte
+}
+
+func (b *Buffer) init() {
+	if b.key == nil {
+		b.key = b.buf[:0]
+	}
+}
+
+// Reset clears the buffer from previous results generated by Key and KeyString.
+func (b *Buffer) Reset() {
+	b.key = b.key[:0]
+}
+
+// Compare returns an integer comparing the two byte slices.
+// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
+func (c *Collator) Compare(a, b []byte) int {
+	// TODO: skip identical prefixes once we have a fast way to detect if a rune is
+	// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
+	c.iter(0).SetInput(a)
+	c.iter(1).SetInput(b)
+	if res := c.compare(); res != 0 {
+		return res
+	}
+	if !c.ignore[colltab.Identity] {
+		return bytes.Compare(a, b)
+	}
+	return 0
+}
+
+// CompareString returns an integer comparing the two strings.
+// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
+func (c *Collator) CompareString(a, b string) int {
+	// TODO: skip identical prefixes once we have a fast way to detect if a rune is
+	// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
+	c.iter(0).SetInputString(a)
+	c.iter(1).SetInputString(b)
+	if res := c.compare(); res != 0 {
+		return res
+	}
+	if !c.ignore[colltab.Identity] {
+		if a < b {
+			return -1
+		} else if a > b {
+			return 1
+		}
+	}
+	return 0
+}
+
+func compareLevel(f func(i *iter) int, a, b *iter) int {
+	a.pce = 0
+	b.pce = 0
+	for {
+		va := f(a)
+		vb := f(b)
+		if va != vb {
+			if va < vb {
+				return -1
+			}
+			return 1
+		} else if va == 0 {
+			break
+		}
+	}
+	return 0
+}
+
+func (c *Collator) compare() int {
+	ia, ib := c.iter(0), c.iter(1)
+	// Process primary level
+	if c.alternate != altShifted {
+		// TODO: implement script reordering
+		if res := compareLevel((*iter).nextPrimary, ia, ib); res != 0 {
+			return res
+		}
+	} else {
+		// TODO: handle shifted
+	}
+	if !c.ignore[colltab.Secondary] {
+		f := (*iter).nextSecondary
+		if c.backwards {
+			f = (*iter).prevSecondary
+		}
+		if res := compareLevel(f, ia, ib); res != 0 {
+			return res
+		}
+	}
+	// TODO: special case handling (Danish?)
+	if !c.ignore[colltab.Tertiary] || c.caseLevel {
+		if res := compareLevel((*iter).nextTertiary, ia, ib); res != 0 {
+			return res
+		}
+		if !c.ignore[colltab.Quaternary] {
+			if res := compareLevel((*iter).nextQuaternary, ia, ib); res != 0 {
+				return res
+			}
+		}
+	}
+	return 0
+}
+
+// Key returns the collation key for str.
+// Passing the buffer buf may avoid memory allocations.
+// The returned slice will point to an allocation in Buffer and will remain
+// valid until the next call to buf.Reset().
+func (c *Collator) Key(buf *Buffer, str []byte) []byte {
+	// See http://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
+	buf.init()
+	return c.key(buf, c.getColElems(str))
+}
+
+// KeyFromString returns the collation key for str.
+// Passing the buffer buf may avoid memory allocations.
+// The returned slice will point to an allocation in Buffer and will retain
+// valid until the next call to buf.ResetKeys().
+func (c *Collator) KeyFromString(buf *Buffer, str string) []byte {
+	// See http://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
+	buf.init()
+	return c.key(buf, c.getColElemsString(str))
+}
+
+func (c *Collator) key(buf *Buffer, w []colltab.Elem) []byte {
+	processWeights(c.alternate, c.t.Top(), w)
+	kn := len(buf.key)
+	c.keyFromElems(buf, w)
+	return buf.key[kn:]
+}
+
+func (c *Collator) getColElems(str []byte) []colltab.Elem {
+	i := c.iter(0)
+	i.SetInput(str)
+	for i.Next() {
+	}
+	return i.Elems
+}
+
+func (c *Collator) getColElemsString(str string) []colltab.Elem {
+	i := c.iter(0)
+	i.SetInputString(str)
+	for i.Next() {
+	}
+	return i.Elems
+}
+
+type iter struct {
+	wa [512]colltab.Elem
+
+	colltab.Iter
+	pce int
+}
+
+func (i *iter) init(c *Collator) {
+	i.Weighter = c.t
+	i.Elems = i.wa[:0]
+}
+
+func (i *iter) nextPrimary() int {
+	for {
+		for ; i.pce < i.N; i.pce++ {
+			if v := i.Elems[i.pce].Primary(); v != 0 {
+				i.pce++
+				return v
+			}
+		}
+		if !i.Next() {
+			return 0
+		}
+	}
+	panic("should not reach here")
+}
+
+func (i *iter) nextSecondary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Secondary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func (i *iter) prevSecondary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[len(i.Elems)-i.pce-1].Secondary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func (i *iter) nextTertiary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Tertiary(); v != 0 {
+			i.pce++
+			return int(v)
+		}
+	}
+	return 0
+}
+
+func (i *iter) nextQuaternary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Quaternary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func appendPrimary(key []byte, p int) []byte {
+	// Convert to variable length encoding; supports up to 23 bits.
+	if p <= 0x7FFF {
+		key = append(key, uint8(p>>8), uint8(p))
+	} else {
+		key = append(key, uint8(p>>16)|0x80, uint8(p>>8), uint8(p))
+	}
+	return key
+}
+
+// keyFromElems converts the weights ws to a compact sequence of bytes.
+// The result will be appended to the byte buffer in buf.
+func (c *Collator) keyFromElems(buf *Buffer, ws []colltab.Elem) {
+	for _, v := range ws {
+		if w := v.Primary(); w > 0 {
+			buf.key = appendPrimary(buf.key, w)
+		}
+	}
+	if !c.ignore[colltab.Secondary] {
+		buf.key = append(buf.key, 0, 0)
+		// TODO: we can use one 0 if we can guarantee that all non-zero weights are > 0xFF.
+		if !c.backwards {
+			for _, v := range ws {
+				if w := v.Secondary(); w > 0 {
+					buf.key = append(buf.key, uint8(w>>8), uint8(w))
+				}
+			}
+		} else {
+			for i := len(ws) - 1; i >= 0; i-- {
+				if w := ws[i].Secondary(); w > 0 {
+					buf.key = append(buf.key, uint8(w>>8), uint8(w))
+				}
+			}
+		}
+	} else if c.caseLevel {
+		buf.key = append(buf.key, 0, 0)
+	}
+	if !c.ignore[colltab.Tertiary] || c.caseLevel {
+		buf.key = append(buf.key, 0, 0)
+		for _, v := range ws {
+			if w := v.Tertiary(); w > 0 {
+				buf.key = append(buf.key, uint8(w))
+			}
+		}
+		// Derive the quaternary weights from the options and other levels.
+		// Note that we represent MaxQuaternary as 0xFF. The first byte of the
+		// representation of a primary weight is always smaller than 0xFF,
+		// so using this single byte value will compare correctly.
+		if !c.ignore[colltab.Quaternary] && c.alternate >= altShifted {
+			if c.alternate == altShiftTrimmed {
+				lastNonFFFF := len(buf.key)
+				buf.key = append(buf.key, 0)
+				for _, v := range ws {
+					if w := v.Quaternary(); w == colltab.MaxQuaternary {
+						buf.key = append(buf.key, 0xFF)
+					} else if w > 0 {
+						buf.key = appendPrimary(buf.key, w)
+						lastNonFFFF = len(buf.key)
+					}
+				}
+				buf.key = buf.key[:lastNonFFFF]
+			} else {
+				buf.key = append(buf.key, 0)
+				for _, v := range ws {
+					if w := v.Quaternary(); w == colltab.MaxQuaternary {
+						buf.key = append(buf.key, 0xFF)
+					} else if w > 0 {
+						buf.key = appendPrimary(buf.key, w)
+					}
+				}
+			}
+		}
+	}
+}
+
+func processWeights(vw alternateHandling, top uint32, wa []colltab.Elem) {
+	ignore := false
+	vtop := int(top)
+	switch vw {
+	case altShifted, altShiftTrimmed:
+		for i := range wa {
+			if p := wa[i].Primary(); p <= vtop && p != 0 {
+				wa[i] = colltab.MakeQuaternary(p)
+				ignore = true
+			} else if p == 0 {
+				if ignore {
+					wa[i] = colltab.Ignore
+				}
+			} else {
+				ignore = false
+			}
+		}
+	case altBlanked:
+		for i := range wa {
+			if p := wa[i].Primary(); p <= vtop && (ignore || p != 0) {
+				wa[i] = colltab.Ignore
+				ignore = true
+			} else {
+				ignore = false
+			}
+		}
+	}
+}

vendor/golang.org/x/text/collate/collate_test.go

@@ -0,0 +1,482 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"bytes"
+	"testing"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+)
+
+type weightsTest struct {
+	opt     opts
+	in, out ColElems
+}
+
+type opts struct {
+	lev int
+	alt alternateHandling
+	top int
+
+	backwards bool
+	caseLevel bool
+}
+
+// ignore returns an initialized boolean array based on the given Level.
+// A negative value means using the default setting of quaternary.
+func ignore(level colltab.Level) (ignore [colltab.NumLevels]bool) {
+	if level < 0 {
+		level = colltab.Quaternary
+	}
+	for i := range ignore {
+		ignore[i] = level < colltab.Level(i)
+	}
+	return ignore
+}
+
+func makeCE(w []int) colltab.Elem {
+	ce, err := colltab.MakeElem(w[0], w[1], w[2], uint8(w[3]))
+	if err != nil {
+		panic(err)
+	}
+	return ce
+}
+
+func (o opts) collator() *Collator {
+	c := &Collator{
+		options: options{
+			ignore:      ignore(colltab.Level(o.lev - 1)),
+			alternate:   o.alt,
+			backwards:   o.backwards,
+			caseLevel:   o.caseLevel,
+			variableTop: uint32(o.top),
+		},
+	}
+	return c
+}
+
+const (
+	maxQ = 0x1FFFFF
+)
+
+func wpq(p, q int) Weights {
+	return W(p, defaults.Secondary, defaults.Tertiary, q)
+}
+
+func wsq(s, q int) Weights {
+	return W(0, s, defaults.Tertiary, q)
+}
+
+func wq(q int) Weights {
+	return W(0, 0, 0, q)
+}
+
+var zero = W(0, 0, 0, 0)
+
+var processTests = []weightsTest{
+	// Shifted
+	{ // simple sequence of non-variables
+		opt: opts{alt: altShifted, top: 100},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)},
+	},
+	{ // first is a variable
+		opt: opts{alt: altShifted, top: 250},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)},
+	},
+	{ // all but first are variable
+		opt: opts{alt: altShifted, top: 999},
+		in:  ColElems{W(1000), W(200), W(300), W(400)},
+		out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)},
+	},
+	{ // first is a modifier
+		opt: opts{alt: altShifted, top: 999},
+		in:  ColElems{W(0, 10), W(1000)},
+		out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)},
+	},
+	{ // primary ignorables
+		opt: opts{alt: altShifted, top: 250},
+		in:  ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)},
+	},
+	{ // secondary ignorables
+		opt: opts{alt: altShifted, top: 250},
+		in:  ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), W(0, 0, 15, maxQ), wpq(400, maxQ)},
+	},
+	{ // tertiary ignorables, no change
+		opt: opts{alt: altShifted, top: 250},
+		in:  ColElems{W(200), zero, W(300), zero, W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)},
+	},
+
+	// ShiftTrimmed (same as Shifted)
+	{ // simple sequence of non-variables
+		opt: opts{alt: altShiftTrimmed, top: 100},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)},
+	},
+	{ // first is a variable
+		opt: opts{alt: altShiftTrimmed, top: 250},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)},
+	},
+	{ // all but first are variable
+		opt: opts{alt: altShiftTrimmed, top: 999},
+		in:  ColElems{W(1000), W(200), W(300), W(400)},
+		out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)},
+	},
+	{ // first is a modifier
+		opt: opts{alt: altShiftTrimmed, top: 999},
+		in:  ColElems{W(0, 10), W(1000)},
+		out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)},
+	},
+	{ // primary ignorables
+		opt: opts{alt: altShiftTrimmed, top: 250},
+		in:  ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)},
+	},
+	{ // secondary ignorables
+		opt: opts{alt: altShiftTrimmed, top: 250},
+		in:  ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), W(0, 0, 15, maxQ), wpq(400, maxQ)},
+	},
+	{ // tertiary ignorables, no change
+		opt: opts{alt: altShiftTrimmed, top: 250},
+		in:  ColElems{W(200), zero, W(300), zero, W(400)},
+		out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)},
+	},
+
+	// Blanked
+	{ // simple sequence of non-variables
+		opt: opts{alt: altBlanked, top: 100},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{W(200), W(300), W(400)},
+	},
+	{ // first is a variable
+		opt: opts{alt: altBlanked, top: 250},
+		in:  ColElems{W(200), W(300), W(400)},
+		out: ColElems{zero, W(300), W(400)},
+	},
+	{ // all but first are variable
+		opt: opts{alt: altBlanked, top: 999},
+		in:  ColElems{W(1000), W(200), W(300), W(400)},
+		out: ColElems{W(1000), zero, zero, zero},
+	},
+	{ // first is a modifier
+		opt: opts{alt: altBlanked, top: 999},
+		in:  ColElems{W(0, 10), W(1000)},
+		out: ColElems{W(0, 10), W(1000)},
+	},
+	{ // primary ignorables
+		opt: opts{alt: altBlanked, top: 250},
+		in:  ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+		out: ColElems{zero, zero, W(300), W(0, 15), W(400)},
+	},
+	{ // secondary ignorables
+		opt: opts{alt: altBlanked, top: 250},
+		in:  ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+		out: ColElems{zero, zero, W(300), W(0, 0, 15), W(400)},
+	},
+	{ // tertiary ignorables, no change
+		opt: opts{alt: altBlanked, top: 250},
+		in:  ColElems{W(200), zero, W(300), zero, W(400)},
+		out: ColElems{zero, zero, W(300), zero, W(400)},
+	},
+
+	// Non-ignorable: input is always equal to output.
+	{ // all but first are variable
+		opt: opts{alt: altNonIgnorable, top: 999},
+		in:  ColElems{W(1000), W(200), W(300), W(400)},
+		out: ColElems{W(1000), W(200), W(300), W(400)},
+	},
+	{ // primary ignorables
+		opt: opts{alt: altNonIgnorable, top: 250},
+		in:  ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+		out: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+	},
+	{ // secondary ignorables
+		opt: opts{alt: altNonIgnorable, top: 250},
+		in:  ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+		out: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+	},
+	{ // tertiary ignorables, no change
+		opt: opts{alt: altNonIgnorable, top: 250},
+		in:  ColElems{W(200), zero, W(300), zero, W(400)},
+		out: ColElems{W(200), zero, W(300), zero, W(400)},
+	},
+}
+
+func TestProcessWeights(t *testing.T) {
+	for i, tt := range processTests {
+		in := convertFromWeights(tt.in)
+		out := convertFromWeights(tt.out)
+		processWeights(tt.opt.alt, uint32(tt.opt.top), in)
+		for j, w := range in {
+			if w != out[j] {
+				t.Errorf("%d: Weights %d was %v; want %v", i, j, w, out[j])
+			}
+		}
+	}
+}
+
+type keyFromElemTest struct {
+	opt opts
+	in  ColElems
+	out []byte
+}
+
+var defS = byte(defaults.Secondary)
+var defT = byte(defaults.Tertiary)
+
+const sep = 0 // separator byte
+
+var keyFromElemTests = []keyFromElemTest{
+	{ // simple primary and secondary weights.
+		opts{alt: altShifted},
+		ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+			sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+		},
+	},
+	{ // same as first, but with zero element that need to be removed
+		opts{alt: altShifted},
+		ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+			sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+		},
+	},
+	{ // same as first, with large primary values
+		opts{alt: altShifted},
+		ColElems{W(0x200), W(0x8000), W(0, 0x30), W(0x12345)},
+		[]byte{0x2, 0, 0x80, 0x80, 0x00, 0x81, 0x23, 0x45, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+			sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+		},
+	},
+	{ // same as first, but with the secondary level backwards
+		opts{alt: altShifted, backwards: true},
+		ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, 0, defS, 0, 0x30, 0, defS, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+			sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+		},
+	},
+	{ // same as first, ignoring quaternary level
+		opts{alt: altShifted, lev: 3},
+		ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+		},
+	},
+	{ // same as first, ignoring tertiary level
+		opts{alt: altShifted, lev: 2},
+		ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+		},
+	},
+	{ // same as first, ignoring secondary level
+		opts{alt: altShifted, lev: 1},
+		ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00},
+	},
+	{ // simple primary and secondary weights.
+		opts{alt: altShiftTrimmed, top: 0x250},
+		ColElems{W(0x300), W(0x200), W(0x7FFF), W(0, 0x30), W(0x800)},
+		[]byte{0x3, 0, 0x7F, 0xFF, 0x8, 0x00, // primary
+			sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+			sep, 0xFF, 0x2, 0, // quaternary
+		},
+	},
+	{ // as first, primary with case level enabled
+		opts{alt: altShifted, lev: 1, caseLevel: true},
+		ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+		[]byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+			sep, sep, // secondary
+			sep, sep, defT, defT, defT, defT, // tertiary
+		},
+	},
+}
+
+func TestKeyFromElems(t *testing.T) {
+	buf := Buffer{}
+	for i, tt := range keyFromElemTests {
+		buf.Reset()
+		in := convertFromWeights(tt.in)
+		processWeights(tt.opt.alt, uint32(tt.opt.top), in)
+		tt.opt.collator().keyFromElems(&buf, in)
+		res := buf.key
+		if len(res) != len(tt.out) {
+			t.Errorf("%d: len(ws) was %d; want %d (%X should be %X)", i, len(res), len(tt.out), res, tt.out)
+		}
+		n := len(res)
+		if len(tt.out) < n {
+			n = len(tt.out)
+		}
+		for j, c := range res[:n] {
+			if c != tt.out[j] {
+				t.Errorf("%d: byte %d was %X; want %X", i, j, c, tt.out[j])
+			}
+		}
+	}
+}
+
+func TestGetColElems(t *testing.T) {
+	for i, tt := range appendNextTests {
+		c, err := makeTable(tt.in)
+		if err != nil {
+			// error is reported in TestAppendNext
+			continue
+		}
+		// Create one large test per table
+		str := make([]byte, 0, 4000)
+		out := ColElems{}
+		for len(str) < 3000 {
+			for _, chk := range tt.chk {
+				str = append(str, chk.in[:chk.n]...)
+				out = append(out, chk.out...)
+			}
+		}
+		for j, chk := range append(tt.chk, check{string(str), len(str), out}) {
+			out := convertFromWeights(chk.out)
+			ce := c.getColElems([]byte(chk.in)[:chk.n])
+			if len(ce) != len(out) {
+				t.Errorf("%d:%d: len(ws) was %d; want %d", i, j, len(ce), len(out))
+				continue
+			}
+			cnt := 0
+			for k, w := range ce {
+				w, _ = colltab.MakeElem(w.Primary(), w.Secondary(), int(w.Tertiary()), 0)
+				if w != out[k] {
+					t.Errorf("%d:%d: Weights %d was %X; want %X", i, j, k, w, out[k])
+					cnt++
+				}
+				if cnt > 10 {
+					break
+				}
+			}
+		}
+	}
+}
+
+type keyTest struct {
+	in  string
+	out []byte
+}
+
+var keyTests = []keyTest{
+	{"abc",
+		[]byte{0, 100, 0, 200, 1, 44, 0, 0, 0, 32, 0, 32, 0, 32, 0, 0, 2, 2, 2, 0, 255, 255, 255},
+	},
+	{"a\u0301",
+		[]byte{0, 102, 0, 0, 0, 32, 0, 0, 2, 0, 255},
+	},
+	{"aaaaa",
+		[]byte{0, 100, 0, 100, 0, 100, 0, 100, 0, 100, 0, 0,
+			0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 0,
+			2, 2, 2, 2, 2, 0,
+			255, 255, 255, 255, 255,
+		},
+	},
+	// Issue 16391: incomplete rune at end of UTF-8 sequence.
+	{"\xc2", []byte{133, 255, 253, 0, 0, 0, 32, 0, 0, 2, 0, 255}},
+	{"\xc2a", []byte{133, 255, 253, 0, 100, 0, 0, 0, 32, 0, 32, 0, 0, 2, 2, 0, 255, 255}},
+}
+
+func TestKey(t *testing.T) {
+	c, _ := makeTable(appendNextTests[4].in)
+	c.alternate = altShifted
+	c.ignore = ignore(colltab.Quaternary)
+	buf := Buffer{}
+	keys1 := [][]byte{}
+	keys2 := [][]byte{}
+	for _, tt := range keyTests {
+		keys1 = append(keys1, c.Key(&buf, []byte(tt.in)))
+		keys2 = append(keys2, c.KeyFromString(&buf, tt.in))
+	}
+	// Separate generation from testing to ensure buffers are not overwritten.
+	for i, tt := range keyTests {
+		if !bytes.Equal(keys1[i], tt.out) {
+			t.Errorf("%d: Key(%q) = %d; want %d", i, tt.in, keys1[i], tt.out)
+		}
+		if !bytes.Equal(keys2[i], tt.out) {
+			t.Errorf("%d: KeyFromString(%q) = %d; want %d", i, tt.in, keys2[i], tt.out)
+		}
+	}
+}
+
+type compareTest struct {
+	a, b string
+	res  int // comparison result
+}
+
+var compareTests = []compareTest{
+	{"a\u0301", "a", 1},
+	{"a\u0301b", "ab", 1},
+	{"a", "a\u0301", -1},
+	{"ab", "a\u0301b", -1},
+	{"bc", "a\u0301c", 1},
+	{"ab", "aB", -1},
+	{"a\u0301", "a\u0301", 0},
+	{"a", "a", 0},
+	// Only clip prefixes of whole runes.
+	{"\u302E", "\u302F", 1},
+	// Don't clip prefixes when last rune of prefix may be part of contraction.
+	{"a\u035E", "a\u0301\u035F", -1},
+	{"a\u0301\u035Fb", "a\u0301\u035F", -1},
+}
+
+func TestCompare(t *testing.T) {
+	c, _ := makeTable(appendNextTests[4].in)
+	for i, tt := range compareTests {
+		if res := c.Compare([]byte(tt.a), []byte(tt.b)); res != tt.res {
+			t.Errorf("%d: Compare(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res)
+		}
+		if res := c.CompareString(tt.a, tt.b); res != tt.res {
+			t.Errorf("%d: CompareString(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res)
+		}
+	}
+}
+
+func TestNumeric(t *testing.T) {
+	c := New(language.English, Loose, Numeric)
+
+	for i, tt := range []struct {
+		a, b string
+		want int
+	}{
+		{"1", "2", -1},
+		{"2", "12", -1},
+		{"２", "１２", -1}, // Fullwidth is sorted as usual.
+		{"₂", "₁₂", 1},  // Subscript is not sorted as numbers.
+		{"②", "①②", 1},  // Circled is not sorted as numbers.
+		{ // Imperial Aramaic, is not sorted as number.
+			"\U00010859",
+			"\U00010858\U00010859",
+			1,
+		},
+		{"12", "2", 1},
+		{"A-1", "A-2", -1},
+		{"A-2", "A-12", -1},
+		{"A-12", "A-2", 1},
+		{"A-0001", "A-1", 0},
+	} {
+		if got := c.CompareString(tt.a, tt.b); got != tt.want {
+			t.Errorf("%d: CompareString(%s, %s) = %d; want %d", i, tt.a, tt.b, got, tt.want)
+		}
+	}
+}

vendor/golang.org/x/text/collate/export_test.go

@@ -0,0 +1,51 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+// Export for testing.
+// TODO: no longer necessary. Remove at some point.
+
+import (
+	"fmt"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+const (
+	defaultSecondary = 0x20
+	defaultTertiary  = 0x2
+)
+
+type Weights struct {
+	Primary, Secondary, Tertiary, Quaternary int
+}
+
+func W(ce ...int) Weights {
+	w := Weights{ce[0], defaultSecondary, defaultTertiary, 0}
+	if len(ce) > 1 {
+		w.Secondary = ce[1]
+	}
+	if len(ce) > 2 {
+		w.Tertiary = ce[2]
+	}
+	if len(ce) > 3 {
+		w.Quaternary = ce[3]
+	}
+	return w
+}
+func (w Weights) String() string {
+	return fmt.Sprintf("[%X.%X.%X.%X]", w.Primary, w.Secondary, w.Tertiary, w.Quaternary)
+}
+
+func convertFromWeights(ws []Weights) []colltab.Elem {
+	out := make([]colltab.Elem, len(ws))
+	for i, w := range ws {
+		out[i], _ = colltab.MakeElem(w.Primary, w.Secondary, w.Tertiary, 0)
+		if out[i] == colltab.Ignore && w.Quaternary > 0 {
+			out[i] = colltab.MakeQuaternary(w.Quaternary)
+		}
+	}
+	return out
+}

vendor/golang.org/x/text/collate/index.go

@@ -0,0 +1,32 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import "golang.org/x/text/internal/colltab"
+
+const blockSize = 64
+
+func getTable(t tableIndex) *colltab.Table {
+	return &colltab.Table{
+		Index: colltab.Trie{
+			Index0:  mainLookup[:][blockSize*t.lookupOffset:],
+			Values0: mainValues[:][blockSize*t.valuesOffset:],
+			Index:   mainLookup[:],
+			Values:  mainValues[:],
+		},
+		ExpandElem:     mainExpandElem[:],
+		ContractTries:  colltab.ContractTrieSet(mainCTEntries[:]),
+		ContractElem:   mainContractElem[:],
+		MaxContractLen: 18,
+		VariableTop:    varTop,
+	}
+}
+
+// tableIndex holds information for constructing a table
+// for a certain locale based on the main table.
+type tableIndex struct {
+	lookupOffset uint32
+	valuesOffset uint32
+}

vendor/golang.org/x/text/collate/maketables.go

@@ -0,0 +1,553 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Collation table generator.
+// Data read from the web.
+
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/collate/build"
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/cldr"
+)
+
+var (
+	test = flag.Bool("test", false,
+		"test existing tables; can be used to compare web data with package data.")
+	short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
+	draft = flag.Bool("draft", false, `Use draft versions, when available.`)
+	tags  = flag.String("tags", "", "build tags to be included after +build directive")
+	pkg   = flag.String("package", "collate",
+		"the name of the package in which the generated file is to be included")
+
+	tables = flagStringSetAllowAll("tables", "collate", "collate,chars",
+		"comma-spearated list of tables to generate.")
+	exclude = flagStringSet("exclude", "zh2", "",
+		"comma-separated list of languages to exclude.")
+	include = flagStringSet("include", "", "",
+		"comma-separated list of languages to include. Include trumps exclude.")
+	// TODO: Not included: unihan gb2312han zhuyin big5han (for size reasons)
+	// TODO: Not included: traditional (buggy for Bengali)
+	types = flagStringSetAllowAll("types", "standard,phonebook,phonetic,reformed,pinyin,stroke", "",
+		"comma-separated list of types that should be included.")
+)
+
+// stringSet implements an ordered set based on a list.  It implements flag.Value
+// to allow a set to be specified as a comma-separated list.
+type stringSet struct {
+	s        []string
+	allowed  *stringSet
+	dirty    bool // needs compaction if true
+	all      bool
+	allowAll bool
+}
+
+func flagStringSet(name, def, allowed, usage string) *stringSet {
+	ss := &stringSet{}
+	if allowed != "" {
+		usage += fmt.Sprintf(" (allowed values: any of %s)", allowed)
+		ss.allowed = &stringSet{}
+		failOnError(ss.allowed.Set(allowed))
+	}
+	ss.Set(def)
+	flag.Var(ss, name, usage)
+	return ss
+}
+
+func flagStringSetAllowAll(name, def, allowed, usage string) *stringSet {
+	ss := &stringSet{allowAll: true}
+	if allowed == "" {
+		flag.Var(ss, name, usage+fmt.Sprintf(` Use "all" to select all.`))
+	} else {
+		ss.allowed = &stringSet{}
+		failOnError(ss.allowed.Set(allowed))
+		flag.Var(ss, name, usage+fmt.Sprintf(` (allowed values: "all" or any of %s)`, allowed))
+	}
+	ss.Set(def)
+	return ss
+}
+
+func (ss stringSet) Len() int {
+	return len(ss.s)
+}
+
+func (ss stringSet) String() string {
+	return strings.Join(ss.s, ",")
+}
+
+func (ss *stringSet) Set(s string) error {
+	if ss.allowAll && s == "all" {
+		ss.s = nil
+		ss.all = true
+		return nil
+	}
+	ss.s = ss.s[:0]
+	for _, s := range strings.Split(s, ",") {
+		if s := strings.TrimSpace(s); s != "" {
+			if ss.allowed != nil && !ss.allowed.contains(s) {
+				return fmt.Errorf("unsupported value %q; must be one of %s", s, ss.allowed)
+			}
+			ss.add(s)
+		}
+	}
+	ss.compact()
+	return nil
+}
+
+func (ss *stringSet) add(s string) {
+	ss.s = append(ss.s, s)
+	ss.dirty = true
+}
+
+func (ss *stringSet) values() []string {
+	ss.compact()
+	return ss.s
+}
+
+func (ss *stringSet) contains(s string) bool {
+	if ss.all {
+		return true
+	}
+	for _, v := range ss.s {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+func (ss *stringSet) compact() {
+	if !ss.dirty {
+		return
+	}
+	a := ss.s
+	sort.Strings(a)
+	k := 0
+	for i := 1; i < len(a); i++ {
+		if a[k] != a[i] {
+			a[k+1] = a[i]
+			k++
+		}
+	}
+	ss.s = a[:k+1]
+	ss.dirty = false
+}
+
+func skipLang(l string) bool {
+	if include.Len() > 0 {
+		return !include.contains(l)
+	}
+	return exclude.contains(l)
+}
+
+// altInclude returns a list of alternatives (for the LDML alt attribute)
+// in order of preference.  An empty string in this list indicates the
+// default entry.
+func altInclude() []string {
+	l := []string{}
+	if *short {
+		l = append(l, "short")
+	}
+	l = append(l, "")
+	// TODO: handle draft using cldr.SetDraftLevel
+	if *draft {
+		l = append(l, "proposed")
+	}
+	return l
+}
+
+func failOnError(e error) {
+	if e != nil {
+		log.Panic(e)
+	}
+}
+
+func openArchive() *zip.Reader {
+	f := gen.OpenCLDRCoreZip()
+	buffer, err := ioutil.ReadAll(f)
+	f.Close()
+	failOnError(err)
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	failOnError(err)
+	return archive
+}
+
+// parseUCA parses a Default Unicode Collation Element Table of the format
+// specified in http://www.unicode.org/reports/tr10/#File_Format.
+// It returns the variable top.
+func parseUCA(builder *build.Builder) {
+	var r io.ReadCloser
+	var err error
+	for _, f := range openArchive().File {
+		if strings.HasSuffix(f.Name, "allkeys_CLDR.txt") {
+			r, err = f.Open()
+		}
+	}
+	if r == nil {
+		log.Fatal("File allkeys_CLDR.txt not found in archive.")
+	}
+	failOnError(err)
+	defer r.Close()
+	scanner := bufio.NewScanner(r)
+	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
+	for i := 1; scanner.Scan(); i++ {
+		line := scanner.Text()
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		if line[0] == '@' {
+			// parse properties
+			switch {
+			case strings.HasPrefix(line[1:], "version "):
+				a := strings.Split(line[1:], " ")
+				if a[1] != gen.UnicodeVersion() {
+					log.Fatalf("incompatible version %s; want %s", a[1], gen.UnicodeVersion())
+				}
+			case strings.HasPrefix(line[1:], "backwards "):
+				log.Fatalf("%d: unsupported option backwards", i)
+			default:
+				log.Printf("%d: unknown option %s", i, line[1:])
+			}
+		} else {
+			// parse entries
+			part := strings.Split(line, " ; ")
+			if len(part) != 2 {
+				log.Fatalf("%d: production rule without ';': %v", i, line)
+			}
+			lhs := []rune{}
+			for _, v := range strings.Split(part[0], " ") {
+				if v == "" {
+					continue
+				}
+				lhs = append(lhs, rune(convHex(i, v)))
+			}
+			var n int
+			var vars []int
+			rhs := [][]int{}
+			for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
+				n += len(m[0])
+				elem := []int{}
+				for _, h := range strings.Split(m[2], ".") {
+					elem = append(elem, convHex(i, h))
+				}
+				if m[1] == "*" {
+					vars = append(vars, i)
+				}
+				rhs = append(rhs, elem)
+			}
+			if len(part[1]) < n+3 || part[1][n+1] != '#' {
+				log.Fatalf("%d: expected comment; found %s", i, part[1][n:])
+			}
+			if *test {
+				testInput.add(string(lhs))
+			}
+			failOnError(builder.Add(lhs, rhs, vars))
+		}
+	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
+}
+
+func convHex(line int, s string) int {
+	r, e := strconv.ParseInt(s, 16, 32)
+	if e != nil {
+		log.Fatalf("%d: %v", line, e)
+	}
+	return int(r)
+}
+
+var testInput = stringSet{}
+
+var charRe = regexp.MustCompile(`&#x([0-9A-F]*);`)
+var tagRe = regexp.MustCompile(`<([a-z_]*)  */>`)
+
+var mainLocales = []string{}
+
+// charsets holds a list of exemplar characters per category.
+type charSets map[string][]string
+
+func (p charSets) fprint(w io.Writer) {
+	fmt.Fprintln(w, "[exN]string{")
+	for i, k := range []string{"", "contractions", "punctuation", "auxiliary", "currencySymbol", "index"} {
+		if set := p[k]; len(set) != 0 {
+			fmt.Fprintf(w, "\t\t%d: %q,\n", i, strings.Join(set, " "))
+		}
+	}
+	fmt.Fprintln(w, "\t},")
+}
+
+var localeChars = make(map[string]charSets)
+
+const exemplarHeader = `
+type exemplarType int
+const (
+	exCharacters exemplarType = iota
+	exContractions
+	exPunctuation
+	exAuxiliary
+	exCurrency
+	exIndex
+	exN
+)
+`
+
+func printExemplarCharacters(w io.Writer) {
+	fmt.Fprintln(w, exemplarHeader)
+	fmt.Fprintln(w, "var exemplarCharacters = map[string][exN]string{")
+	for _, loc := range mainLocales {
+		fmt.Fprintf(w, "\t%q: ", loc)
+		localeChars[loc].fprint(w)
+	}
+	fmt.Fprintln(w, "}")
+}
+
+func decodeCLDR(d *cldr.Decoder) *cldr.CLDR {
+	r := gen.OpenCLDRCoreZip()
+	data, err := d.DecodeZip(r)
+	failOnError(err)
+	return data
+}
+
+// parseMain parses XML files in the main directory of the CLDR core.zip file.
+func parseMain() {
+	d := &cldr.Decoder{}
+	d.SetDirFilter("main")
+	d.SetSectionFilter("characters")
+	data := decodeCLDR(d)
+	for _, loc := range data.Locales() {
+		x := data.RawLDML(loc)
+		if skipLang(x.Identity.Language.Type) {
+			continue
+		}
+		if x.Characters != nil {
+			x, _ = data.LDML(loc)
+			loc = language.Make(loc).String()
+			for _, ec := range x.Characters.ExemplarCharacters {
+				if ec.Draft != "" {
+					continue
+				}
+				if _, ok := localeChars[loc]; !ok {
+					mainLocales = append(mainLocales, loc)
+					localeChars[loc] = make(charSets)
+				}
+				localeChars[loc][ec.Type] = parseCharacters(ec.Data())
+			}
+		}
+	}
+}
+
+func parseCharacters(chars string) []string {
+	parseSingle := func(s string) (r rune, tail string, escaped bool) {
+		if s[0] == '\\' {
+			return rune(s[1]), s[2:], true
+		}
+		r, sz := utf8.DecodeRuneInString(s)
+		return r, s[sz:], false
+	}
+	chars = strings.TrimSpace(chars)
+	if n := len(chars) - 1; chars[n] == ']' && chars[0] == '[' {
+		chars = chars[1:n]
+	}
+	list := []string{}
+	var r, last, end rune
+	for len(chars) > 0 {
+		if chars[0] == '{' { // character sequence
+			buf := []rune{}
+			for chars = chars[1:]; len(chars) > 0; {
+				r, chars, _ = parseSingle(chars)
+				if r == '}' {
+					break
+				}
+				if r == ' ' {
+					log.Fatalf("space not supported in sequence %q", chars)
+				}
+				buf = append(buf, r)
+			}
+			list = append(list, string(buf))
+			last = 0
+		} else { // single character
+			escaped := false
+			r, chars, escaped = parseSingle(chars)
+			if r != ' ' {
+				if r == '-' && !escaped {
+					if last == 0 {
+						log.Fatal("'-' should be preceded by a character")
+					}
+					end, chars, _ = parseSingle(chars)
+					for ; last <= end; last++ {
+						list = append(list, string(last))
+					}
+					last = 0
+				} else {
+					list = append(list, string(r))
+					last = r
+				}
+			}
+		}
+	}
+	return list
+}
+
+var fileRe = regexp.MustCompile(`.*/collation/(.*)\.xml`)
+
+// typeMap translates legacy type keys to their BCP47 equivalent.
+var typeMap = map[string]string{
+	"phonebook":   "phonebk",
+	"traditional": "trad",
+}
+
+// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
+func parseCollation(b *build.Builder) {
+	d := &cldr.Decoder{}
+	d.SetDirFilter("collation")
+	data := decodeCLDR(d)
+	for _, loc := range data.Locales() {
+		x, err := data.LDML(loc)
+		failOnError(err)
+		if skipLang(x.Identity.Language.Type) {
+			continue
+		}
+		cs := x.Collations.Collation
+		sl := cldr.MakeSlice(&cs)
+		if len(types.s) == 0 {
+			sl.SelectAnyOf("type", x.Collations.Default())
+		} else if !types.all {
+			sl.SelectAnyOf("type", types.s...)
+		}
+		sl.SelectOnePerGroup("alt", altInclude())
+
+		for _, c := range cs {
+			id, err := language.Parse(loc)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "invalid locale: %q", err)
+				continue
+			}
+			// Support both old- and new-style defaults.
+			d := c.Type
+			if x.Collations.DefaultCollation == nil {
+				d = x.Collations.Default()
+			} else {
+				d = x.Collations.DefaultCollation.Data()
+			}
+			// We assume tables are being built either for search or collation,
+			// but not both. For search the default is always "search".
+			if d != c.Type && c.Type != "search" {
+				typ := c.Type
+				if len(c.Type) > 8 {
+					typ = typeMap[c.Type]
+				}
+				id, err = id.SetTypeForKey("co", typ)
+				failOnError(err)
+			}
+			t := b.Tailoring(id)
+			c.Process(processor{t})
+		}
+	}
+}
+
+type processor struct {
+	t *build.Tailoring
+}
+
+func (p processor) Reset(anchor string, before int) (err error) {
+	if before != 0 {
+		err = p.t.SetAnchorBefore(anchor)
+	} else {
+		err = p.t.SetAnchor(anchor)
+	}
+	failOnError(err)
+	return nil
+}
+
+func (p processor) Insert(level int, str, context, extend string) error {
+	str = context + str
+	if *test {
+		testInput.add(str)
+	}
+	// TODO: mimic bug in old maketables: remove.
+	err := p.t.Insert(colltab.Level(level-1), str, context+extend)
+	failOnError(err)
+	return nil
+}
+
+func (p processor) Index(id string) {
+}
+
+func testCollator(c *collate.Collator) {
+	c0 := collate.New(language.Und)
+
+	// iterator over all characters for all locales and check
+	// whether Key is equal.
+	buf := collate.Buffer{}
+
+	// Add all common and not too uncommon runes to the test set.
+	for i := rune(0); i < 0x30000; i++ {
+		testInput.add(string(i))
+	}
+	for i := rune(0xE0000); i < 0xF0000; i++ {
+		testInput.add(string(i))
+	}
+	for _, str := range testInput.values() {
+		k0 := c0.KeyFromString(&buf, str)
+		k := c.KeyFromString(&buf, str)
+		if !bytes.Equal(k0, k) {
+			failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
+		}
+		buf.Reset()
+	}
+	fmt.Println("PASS")
+}
+
+func main() {
+	gen.Init()
+	b := build.NewBuilder()
+	parseUCA(b)
+	if tables.contains("chars") {
+		parseMain()
+	}
+	parseCollation(b)
+
+	c, err := b.Build()
+	failOnError(err)
+
+	if *test {
+		testCollator(collate.NewFromTable(c))
+	} else {
+		w := &bytes.Buffer{}
+
+		gen.WriteUnicodeVersion(w)
+		gen.WriteCLDRVersion(w)
+
+		if tables.contains("collate") {
+			_, err = b.Print(w)
+			failOnError(err)
+		}
+		if tables.contains("chars") {
+			printExemplarCharacters(w)
+		}
+		gen.WriteGoFile("tables.go", *pkg, w.Bytes())
+	}
+}

vendor/golang.org/x/text/collate/option.go

@@ -0,0 +1,239 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"sort"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+// newCollator creates a new collator with default options configured.
+func newCollator(t colltab.Weighter) *Collator {
+	// Initialize a collator with default options.
+	c := &Collator{
+		options: options{
+			ignore: [colltab.NumLevels]bool{
+				colltab.Quaternary: true,
+				colltab.Identity:   true,
+			},
+			f: norm.NFD,
+			t: t,
+		},
+	}
+
+	// TODO: store vt in tags or remove.
+	c.variableTop = t.Top()
+
+	return c
+}
+
+// An Option is used to change the behavior of a Collator. Options override the
+// settings passed through the locale identifier.
+type Option struct {
+	priority int
+	f        func(o *options)
+}
+
+type prioritizedOptions []Option
+
+func (p prioritizedOptions) Len() int {
+	return len(p)
+}
+
+func (p prioritizedOptions) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (p prioritizedOptions) Less(i, j int) bool {
+	return p[i].priority < p[j].priority
+}
+
+type options struct {
+	// ignore specifies which levels to ignore.
+	ignore [colltab.NumLevels]bool
+
+	// caseLevel is true if there is an additional level of case matching
+	// between the secondary and tertiary levels.
+	caseLevel bool
+
+	// backwards specifies the order of sorting at the secondary level.
+	// This option exists predominantly to support reverse sorting of accents in French.
+	backwards bool
+
+	// numeric specifies whether any sequence of decimal digits (category is Nd)
+	// is sorted at a primary level with its numeric value.
+	// For example, "A-21" < "A-123".
+	// This option is set by wrapping the main Weighter with NewNumericWeighter.
+	numeric bool
+
+	// alternate specifies an alternative handling of variables.
+	alternate alternateHandling
+
+	// variableTop is the largest primary value that is considered to be
+	// variable.
+	variableTop uint32
+
+	t colltab.Weighter
+
+	f norm.Form
+}
+
+func (o *options) setOptions(opts []Option) {
+	sort.Sort(prioritizedOptions(opts))
+	for _, x := range opts {
+		x.f(o)
+	}
+}
+
+// OptionsFromTag extracts the BCP47 collation options from the tag and
+// configures a collator accordingly. These options are set before any other
+// option.
+func OptionsFromTag(t language.Tag) Option {
+	return Option{0, func(o *options) {
+		o.setFromTag(t)
+	}}
+}
+
+func (o *options) setFromTag(t language.Tag) {
+	o.caseLevel = ldmlBool(t, o.caseLevel, "kc")
+	o.backwards = ldmlBool(t, o.backwards, "kb")
+	o.numeric = ldmlBool(t, o.numeric, "kn")
+
+	// Extract settings from the BCP47 u extension.
+	switch t.TypeForKey("ks") { // strength
+	case "level1":
+		o.ignore[colltab.Secondary] = true
+		o.ignore[colltab.Tertiary] = true
+	case "level2":
+		o.ignore[colltab.Tertiary] = true
+	case "level3", "":
+		// The default.
+	case "level4":
+		o.ignore[colltab.Quaternary] = false
+	case "identic":
+		o.ignore[colltab.Quaternary] = false
+		o.ignore[colltab.Identity] = false
+	}
+
+	switch t.TypeForKey("ka") {
+	case "shifted":
+		o.alternate = altShifted
+	// The following two types are not official BCP47, but we support them to
+	// give access to this otherwise hidden functionality. The name blanked is
+	// derived from the LDML name blanked and posix reflects the main use of
+	// the shift-trimmed option.
+	case "blanked":
+		o.alternate = altBlanked
+	case "posix":
+		o.alternate = altShiftTrimmed
+	}
+
+	// TODO: caseFirst ("kf"), reorder ("kr"), and maybe variableTop ("vt").
+
+	// Not used:
+	// - normalization ("kk", not necessary for this implementation)
+	// - hiraganaQuatenary ("kh", obsolete)
+}
+
+func ldmlBool(t language.Tag, old bool, key string) bool {
+	switch t.TypeForKey(key) {
+	case "true":
+		return true
+	case "false":
+		return false
+	default:
+		return old
+	}
+}
+
+var (
+	// IgnoreCase sets case-insensitive comparison.
+	IgnoreCase Option = ignoreCase
+	ignoreCase        = Option{3, ignoreCaseF}
+
+	// IgnoreDiacritics causes diacritical marks to be ignored. ("o" == "ö").
+	IgnoreDiacritics Option = ignoreDiacritics
+	ignoreDiacritics        = Option{3, ignoreDiacriticsF}
+
+	// IgnoreWidth causes full-width characters to match their half-width
+	// equivalents.
+	IgnoreWidth Option = ignoreWidth
+	ignoreWidth        = Option{2, ignoreWidthF}
+
+	// Loose sets the collator to ignore diacritics, case and weight.
+	Loose Option = loose
+	loose        = Option{4, looseF}
+
+	// Force ordering if strings are equivalent but not equal.
+	Force Option = force
+	force        = Option{5, forceF}
+
+	// Numeric specifies that numbers should sort numerically ("2" < "12").
+	Numeric Option = numeric
+	numeric        = Option{5, numericF}
+)
+
+func ignoreWidthF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = true
+}
+
+func ignoreDiacriticsF(o *options) {
+	o.ignore[colltab.Secondary] = true
+}
+
+func ignoreCaseF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = false
+}
+
+func looseF(o *options) {
+	ignoreWidthF(o)
+	ignoreDiacriticsF(o)
+	ignoreCaseF(o)
+}
+
+func forceF(o *options) {
+	o.ignore[colltab.Identity] = false
+}
+
+func numericF(o *options) { o.numeric = true }
+
+// Reorder overrides the pre-defined ordering of scripts and character sets.
+func Reorder(s ...string) Option {
+	// TODO: need fractional weights to implement this.
+	panic("TODO: implement")
+}
+
+// TODO: consider making these public again. These options cannot be fully
+// specified in BCP47, so an API interface seems warranted. Still a higher-level
+// interface would be nice (e.g. a POSIX option for enabling altShiftTrimmed)
+
+// alternateHandling identifies the various ways in which variables are handled.
+// A rune with a primary weight lower than the variable top is considered a
+// variable.
+// See http://www.unicode.org/reports/tr10/#Variable_Weighting for details.
+type alternateHandling int
+
+const (
+	// altNonIgnorable turns off special handling of variables.
+	altNonIgnorable alternateHandling = iota
+
+	// altBlanked sets variables and all subsequent primary ignorables to be
+	// ignorable at all levels. This is identical to removing all variables
+	// and subsequent primary ignorables from the input.
+	altBlanked
+
+	// altShifted sets variables to be ignorable for levels one through three and
+	// adds a fourth level based on the values of the ignored levels.
+	altShifted
+
+	// altShiftTrimmed is a slight variant of altShifted that is used to
+	// emulate POSIX.
+	altShiftTrimmed
+)

vendor/golang.org/x/text/collate/option_test.go

@@ -0,0 +1,209 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package collate
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+)
+
+var (
+	defaultIgnore = ignore(colltab.Tertiary)
+	defaultTable  = getTable(locales[0])
+)
+
+func TestOptions(t *testing.T) {
+	for i, tt := range []struct {
+		in  []Option
+		out options
+	}{
+		0: {
+			out: options{
+				ignore: defaultIgnore,
+			},
+		},
+		1: {
+			in: []Option{IgnoreDiacritics},
+			out: options{
+				ignore: [colltab.NumLevels]bool{false, true, false, true, true},
+			},
+		},
+		2: {
+			in: []Option{IgnoreCase, IgnoreDiacritics},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		3: {
+			in: []Option{ignoreDiacritics, IgnoreWidth},
+			out: options{
+				ignore:    ignore(colltab.Primary),
+				caseLevel: true,
+			},
+		},
+		4: {
+			in: []Option{IgnoreWidth, ignoreDiacritics},
+			out: options{
+				ignore:    ignore(colltab.Primary),
+				caseLevel: true,
+			},
+		},
+		5: {
+			in: []Option{IgnoreCase, IgnoreWidth},
+			out: options{
+				ignore: ignore(colltab.Secondary),
+			},
+		},
+		6: {
+			in: []Option{IgnoreCase, IgnoreWidth, Loose},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		7: {
+			in: []Option{Force, IgnoreCase, IgnoreWidth, Loose},
+			out: options{
+				ignore: [colltab.NumLevels]bool{false, true, true, true, false},
+			},
+		},
+		8: {
+			in: []Option{IgnoreDiacritics, IgnoreCase},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		9: {
+			in: []Option{Numeric},
+			out: options{
+				ignore:  defaultIgnore,
+				numeric: true,
+			},
+		},
+		10: {
+			in: []Option{OptionsFromTag(language.MustParse("und-u-ks-level1"))},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		11: {
+			in: []Option{OptionsFromTag(language.MustParse("und-u-ks-level4"))},
+			out: options{
+				ignore: ignore(colltab.Quaternary),
+			},
+		},
+		12: {
+			in:  []Option{OptionsFromTag(language.MustParse("und-u-ks-identic"))},
+			out: options{},
+		},
+		13: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				caseLevel: true,
+				backwards: true,
+				numeric:   true,
+			},
+		},
+		14: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+				OptionsFromTag(language.MustParse("und-u-kn-false-kb-false-kc-false")),
+			},
+			out: options{
+				ignore: defaultIgnore,
+			},
+		},
+		15: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+				OptionsFromTag(language.MustParse("und-u-kn-foo-kb-foo-kc-foo")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				caseLevel: true,
+				backwards: true,
+				numeric:   true,
+			},
+		},
+		16: { // Normal options take precedence over tag options.
+			in: []Option{
+				Numeric, IgnoreCase,
+				OptionsFromTag(language.MustParse("und-u-kn-false-kc-true")),
+			},
+			out: options{
+				ignore:    ignore(colltab.Secondary),
+				caseLevel: false,
+				numeric:   true,
+			},
+		},
+		17: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-shifted")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altShifted,
+			},
+		},
+		18: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-blanked")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altBlanked,
+			},
+		},
+		19: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-posix")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altShiftTrimmed,
+			},
+		},
+	} {
+		c := newCollator(defaultTable)
+		c.t = nil
+		c.variableTop = 0
+		c.f = 0
+
+		c.setOptions(tt.in)
+		if !reflect.DeepEqual(c.options, tt.out) {
+			t.Errorf("%d: got %v; want %v", i, c.options, tt.out)
+		}
+	}
+}
+
+func TestAlternateSortTypes(t *testing.T) {
+	testCases := []struct {
+		lang string
+		in   []string
+		want []string
+	}{{
+		lang: "zh,cmn,zh-Hant-u-co-pinyin,zh-HK-u-co-pinyin,zh-pinyin",
+		in:   []string{"爸爸", "妈妈", "儿子", "女儿"},
+		want: []string{"爸爸", "儿子", "妈妈", "女儿"},
+	}, {
+		lang: "zh-Hant,zh-u-co-stroke,zh-Hant-u-co-stroke",
+		in:   []string{"爸爸", "妈妈", "儿子", "女儿"},
+		want: []string{"儿子", "女儿", "妈妈", "爸爸"},
+	}}
+	for _, tc := range testCases {
+		for _, tag := range strings.Split(tc.lang, ",") {
+			got := append([]string{}, tc.in...)
+			New(language.MustParse(tag)).SortStrings(got)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("New(%s).SortStrings(%v) = %v; want %v", tag, tc.in, got, tc.want)
+			}
+		}
+	}
+}

vendor/golang.org/x/text/collate/reg_test.go

@@ -0,0 +1,230 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"golang.org/x/text/collate/build"
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/language"
+)
+
+var long = flag.Bool("long", false,
+	"run time-consuming tests, such as tests that fetch data online")
+
+// This regression test runs tests for the test files in CollationTest.zip
+// (taken from http://www.unicode.org/Public/UCA/<gen.UnicodeVersion()>/).
+//
+// The test files have the following form:
+// # header
+// 0009 0021;	# ('\u0009') <CHARACTER TABULATION>	[| | | 0201 025E]
+// 0009 003F;	# ('\u0009') <CHARACTER TABULATION>	[| | | 0201 0263]
+// 000A 0021;	# ('\u000A') <LINE FEED (LF)>	[| | | 0202 025E]
+// 000A 003F;	# ('\u000A') <LINE FEED (LF)>	[| | | 0202 0263]
+//
+// The part before the semicolon is the hex representation of a sequence
+// of runes. After the hash mark is a comment. The strings
+// represented by rune sequence are in the file in sorted order, as
+// defined by the DUCET.
+
+type Test struct {
+	name    string
+	str     [][]byte
+	comment []string
+}
+
+var versionRe = regexp.MustCompile(`# UCA Version: (.*)\n?$`)
+var testRe = regexp.MustCompile(`^([\dA-F ]+);.*# (.*)\n?$`)
+
+func TestCollation(t *testing.T) {
+	if !gen.IsLocal() && !*long {
+		t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source")
+	}
+	t.Skip("must first update to new file format to support test")
+	for _, test := range loadTestData() {
+		doTest(t, test)
+	}
+}
+
+func Error(e error) {
+	if e != nil {
+		log.Fatal(e)
+	}
+}
+
+// parseUCA parses a Default Unicode Collation Element Table of the format
+// specified in http://www.unicode.org/reports/tr10/#File_Format.
+// It returns the variable top.
+func parseUCA(builder *build.Builder) {
+	r := gen.OpenUnicodeFile("UCA", "", "allkeys.txt")
+	defer r.Close()
+	input := bufio.NewReader(r)
+	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
+	for i := 1; true; i++ {
+		l, prefix, err := input.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		Error(err)
+		line := string(l)
+		if prefix {
+			log.Fatalf("%d: buffer overflow", i)
+		}
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		if line[0] == '@' {
+			if strings.HasPrefix(line[1:], "version ") {
+				if v := strings.Split(line[1:], " ")[1]; v != gen.UnicodeVersion() {
+					log.Fatalf("incompatible version %s; want %s", v, gen.UnicodeVersion())
+				}
+			}
+		} else {
+			// parse entries
+			part := strings.Split(line, " ; ")
+			if len(part) != 2 {
+				log.Fatalf("%d: production rule without ';': %v", i, line)
+			}
+			lhs := []rune{}
+			for _, v := range strings.Split(part[0], " ") {
+				if v != "" {
+					lhs = append(lhs, rune(convHex(i, v)))
+				}
+			}
+			vars := []int{}
+			rhs := [][]int{}
+			for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
+				if m[1] == "*" {
+					vars = append(vars, i)
+				}
+				elem := []int{}
+				for _, h := range strings.Split(m[2], ".") {
+					elem = append(elem, convHex(i, h))
+				}
+				rhs = append(rhs, elem)
+			}
+			builder.Add(lhs, rhs, vars)
+		}
+	}
+}
+
+func convHex(line int, s string) int {
+	r, e := strconv.ParseInt(s, 16, 32)
+	if e != nil {
+		log.Fatalf("%d: %v", line, e)
+	}
+	return int(r)
+}
+
+func loadTestData() []Test {
+	f := gen.OpenUnicodeFile("UCA", "", "CollationTest.zip")
+	buffer, err := ioutil.ReadAll(f)
+	f.Close()
+	Error(err)
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	Error(err)
+	tests := []Test{}
+	for _, f := range archive.File {
+		// Skip the short versions, which are simply duplicates of the long versions.
+		if strings.Contains(f.Name, "SHORT") || f.FileInfo().IsDir() {
+			continue
+		}
+		ff, err := f.Open()
+		Error(err)
+		defer ff.Close()
+		scanner := bufio.NewScanner(ff)
+		test := Test{name: path.Base(f.Name)}
+		for scanner.Scan() {
+			line := scanner.Text()
+			if len(line) <= 1 || line[0] == '#' {
+				if m := versionRe.FindStringSubmatch(line); m != nil {
+					if m[1] != gen.UnicodeVersion() {
+						log.Printf("warning:%s: version is %s; want %s", f.Name, m[1], gen.UnicodeVersion())
+					}
+				}
+				continue
+			}
+			m := testRe.FindStringSubmatch(line)
+			if m == nil || len(m) < 3 {
+				log.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+			}
+			str := []byte{}
+			// In the regression test data (unpaired) surrogates are assigned a weight
+			// corresponding to their code point value.  However, utf8.DecodeRune,
+			// which is used to compute the implicit weight, assigns FFFD to surrogates.
+			// We therefore skip tests with surrogates.  This skips about 35 entries
+			// per test.
+			valid := true
+			for _, split := range strings.Split(m[1], " ") {
+				r, err := strconv.ParseUint(split, 16, 64)
+				Error(err)
+				valid = valid && utf8.ValidRune(rune(r))
+				str = append(str, string(rune(r))...)
+			}
+			if valid {
+				test.str = append(test.str, str)
+				test.comment = append(test.comment, m[2])
+			}
+		}
+		if scanner.Err() != nil {
+			log.Fatal(scanner.Err())
+		}
+		tests = append(tests, test)
+	}
+	return tests
+}
+
+var errorCount int
+
+func runes(b []byte) []rune {
+	return []rune(string(b))
+}
+
+var shifted = language.MustParse("und-u-ka-shifted-ks-level4")
+
+func doTest(t *testing.T, tc Test) {
+	bld := build.NewBuilder()
+	parseUCA(bld)
+	w, err := bld.Build()
+	Error(err)
+	var tag language.Tag
+	if !strings.Contains(tc.name, "NON_IGNOR") {
+		tag = shifted
+	}
+	c := NewFromTable(w, OptionsFromTag(tag))
+	b := &Buffer{}
+	prev := tc.str[0]
+	for i := 1; i < len(tc.str); i++ {
+		b.Reset()
+		s := tc.str[i]
+		ka := c.Key(b, prev)
+		kb := c.Key(b, s)
+		if r := bytes.Compare(ka, kb); r == 1 {
+			t.Errorf("%s:%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", tc.name, i, []rune(string(prev)), []rune(string(s)), ka, kb, r)
+			prev = s
+			continue
+		}
+		if r := c.Compare(prev, s); r == 1 {
+			t.Errorf("%s:%d: Compare(%.4X, %.4X) == %d; want -1 or 0", tc.name, i, runes(prev), runes(s), r)
+		}
+		if r := c.Compare(s, prev); r == -1 {
+			t.Errorf("%s:%d: Compare(%.4X, %.4X) == %d; want 1 or 0", tc.name, i, runes(s), runes(prev), r)
+		}
+		prev = s
+	}
+}

vendor/golang.org/x/text/collate/sort.go

@@ -0,0 +1,81 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"bytes"
+	"sort"
+)
+
+const (
+	maxSortBuffer  = 40960
+	maxSortEntries = 4096
+)
+
+type swapper interface {
+	Swap(i, j int)
+}
+
+type sorter struct {
+	buf  *Buffer
+	keys [][]byte
+	src  swapper
+}
+
+func (s *sorter) init(n int) {
+	if s.buf == nil {
+		s.buf = &Buffer{}
+		s.buf.init()
+	}
+	if cap(s.keys) < n {
+		s.keys = make([][]byte, n)
+	}
+	s.keys = s.keys[0:n]
+}
+
+func (s *sorter) sort(src swapper) {
+	s.src = src
+	sort.Sort(s)
+}
+
+func (s sorter) Len() int {
+	return len(s.keys)
+}
+
+func (s sorter) Less(i, j int) bool {
+	return bytes.Compare(s.keys[i], s.keys[j]) == -1
+}
+
+func (s sorter) Swap(i, j int) {
+	s.keys[i], s.keys[j] = s.keys[j], s.keys[i]
+	s.src.Swap(i, j)
+}
+
+// A Lister can be sorted by Collator's Sort method.
+type Lister interface {
+	Len() int
+	Swap(i, j int)
+	// Bytes returns the bytes of the text at index i.
+	Bytes(i int) []byte
+}
+
+// Sort uses sort.Sort to sort the strings represented by x using the rules of c.
+func (c *Collator) Sort(x Lister) {
+	n := x.Len()
+	c.sorter.init(n)
+	for i := 0; i < n; i++ {
+		c.sorter.keys[i] = c.Key(c.sorter.buf, x.Bytes(i))
+	}
+	c.sorter.sort(x)
+}
+
+// SortStrings uses sort.Sort to sort the strings in x using the rules of c.
+func (c *Collator) SortStrings(x []string) {
+	c.sorter.init(len(x))
+	for i, s := range x {
+		c.sorter.keys[i] = c.KeyFromString(c.sorter.buf, s)
+	}
+	c.sorter.sort(sort.StringSlice(x))
+}

vendor/golang.org/x/text/collate/sort_test.go

@@ -0,0 +1,55 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate_test
+
+import (
+	"fmt"
+	"testing"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+)
+
+func ExampleCollator_Strings() {
+	c := collate.New(language.Und)
+	strings := []string{
+		"ad",
+		"ab",
+		"äb",
+		"ac",
+	}
+	c.SortStrings(strings)
+	fmt.Println(strings)
+	// Output: [ab äb ac ad]
+}
+
+type sorter []string
+
+func (s sorter) Len() int {
+	return len(s)
+}
+
+func (s sorter) Swap(i, j int) {
+	s[j], s[i] = s[i], s[j]
+}
+
+func (s sorter) Bytes(i int) []byte {
+	return []byte(s[i])
+}
+
+func TestSort(t *testing.T) {
+	c := collate.New(language.English)
+	strings := []string{
+		"bcd",
+		"abc",
+		"ddd",
+	}
+	c.Sort(sorter(strings))
+	res := fmt.Sprint(strings)
+	want := "[abc bcd ddd]"
+	if res != want {
+		t.Errorf("found %s; want %s", res, want)
+	}
+}

vendor/golang.org/x/text/collate/table_test.go

@@ -0,0 +1,291 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"testing"
+
+	"golang.org/x/text/collate/build"
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/unicode/norm"
+)
+
+type ColElems []Weights
+
+type input struct {
+	str string
+	ces [][]int
+}
+
+type check struct {
+	in  string
+	n   int
+	out ColElems
+}
+
+type tableTest struct {
+	in  []input
+	chk []check
+}
+
+func w(ce ...int) Weights {
+	return W(ce...)
+}
+
+var defaults = w(0)
+
+func pt(p, t int) []int {
+	return []int{p, defaults.Secondary, t}
+}
+
+func makeTable(in []input) (*Collator, error) {
+	b := build.NewBuilder()
+	for _, r := range in {
+		if e := b.Add([]rune(r.str), r.ces, nil); e != nil {
+			panic(e)
+		}
+	}
+	t, err := b.Build()
+	if err != nil {
+		return nil, err
+	}
+	return NewFromTable(t), nil
+}
+
+// modSeq holds a seqeunce of modifiers in increasing order of CCC long enough
+// to cause a segment overflow if not handled correctly. The last rune in this
+// list has a CCC of 214.
+var modSeq = []rune{
+	0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, 0x05B8, 0x05B9, 0x05BB,
+	0x05BC, 0x05BD, 0x05BF, 0x05C1, 0x05C2, 0xFB1E, 0x064B, 0x064C, 0x064D, 0x064E,
+	0x064F, 0x0650, 0x0651, 0x0652, 0x0670, 0x0711, 0x0C55, 0x0C56, 0x0E38, 0x0E48,
+	0x0EB8, 0x0EC8, 0x0F71, 0x0F72, 0x0F74, 0x0321, 0x1DCE,
+}
+
+var mods []input
+var modW = func() ColElems {
+	ws := ColElems{}
+	for _, r := range modSeq {
+		rune := norm.NFC.PropertiesString(string(r))
+		ws = append(ws, w(0, int(rune.CCC())))
+		mods = append(mods, input{string(r), [][]int{{0, int(rune.CCC())}}})
+	}
+	return ws
+}()
+
+var appendNextTests = []tableTest{
+	{ // test getWeights
+		[]input{
+			{"a", [][]int{{100}}},
+			{"b", [][]int{{105}}},
+			{"c", [][]int{{110}}},
+			{"ß", [][]int{{120}}},
+		},
+		[]check{
+			{"a", 1, ColElems{w(100)}},
+			{"b", 1, ColElems{w(105)}},
+			{"c", 1, ColElems{w(110)}},
+			{"d", 1, ColElems{w(0x50064)}},
+			{"ab", 1, ColElems{w(100)}},
+			{"bc", 1, ColElems{w(105)}},
+			{"dd", 1, ColElems{w(0x50064)}},
+			{"ß", 2, ColElems{w(120)}},
+		},
+	},
+	{ // test expansion
+		[]input{
+			{"u", [][]int{{100}}},
+			{"U", [][]int{{100}, {0, 25}}},
+			{"w", [][]int{{100}, {100}}},
+			{"W", [][]int{{100}, {0, 25}, {100}, {0, 25}}},
+		},
+		[]check{
+			{"u", 1, ColElems{w(100)}},
+			{"U", 1, ColElems{w(100), w(0, 25)}},
+			{"w", 1, ColElems{w(100), w(100)}},
+			{"W", 1, ColElems{w(100), w(0, 25), w(100), w(0, 25)}},
+		},
+	},
+	{ // test decompose
+		[]input{
+			{"D", [][]int{pt(104, 8)}},
+			{"z", [][]int{pt(130, 8)}},
+			{"\u030C", [][]int{{0, 40}}},                               // Caron
+			{"\u01C5", [][]int{pt(104, 9), pt(130, 4), {0, 40, 0x1F}}}, // Dž = D+z+caron
+		},
+		[]check{
+			{"\u01C5", 2, ColElems{w(pt(104, 9)...), w(pt(130, 4)...), w(0, 40, 0x1F)}},
+		},
+	},
+	{ // test basic contraction
+		[]input{
+			{"a", [][]int{{100}}},
+			{"ab", [][]int{{101}}},
+			{"aab", [][]int{{101}, {101}}},
+			{"abc", [][]int{{102}}},
+			{"b", [][]int{{200}}},
+			{"c", [][]int{{300}}},
+			{"d", [][]int{{400}}},
+		},
+		[]check{
+			{"a", 1, ColElems{w(100)}},
+			{"aa", 1, ColElems{w(100)}},
+			{"aac", 1, ColElems{w(100)}},
+			{"d", 1, ColElems{w(400)}},
+			{"ab", 2, ColElems{w(101)}},
+			{"abb", 2, ColElems{w(101)}},
+			{"aab", 3, ColElems{w(101), w(101)}},
+			{"aaba", 3, ColElems{w(101), w(101)}},
+			{"abc", 3, ColElems{w(102)}},
+			{"abcd", 3, ColElems{w(102)}},
+		},
+	},
+	{ // test discontinuous contraction
+		append(mods, []input{
+			// modifiers; secondary weight equals ccc
+			{"\u0316", [][]int{{0, 220}}},
+			{"\u0317", [][]int{{0, 220}, {0, 220}}},
+			{"\u302D", [][]int{{0, 222}}},
+			{"\u302E", [][]int{{0, 225}}}, // used as starter
+			{"\u302F", [][]int{{0, 224}}}, // used as starter
+			{"\u18A9", [][]int{{0, 228}}},
+			{"\u0300", [][]int{{0, 230}}},
+			{"\u0301", [][]int{{0, 230}}},
+			{"\u0315", [][]int{{0, 232}}},
+			{"\u031A", [][]int{{0, 232}}},
+			{"\u035C", [][]int{{0, 233}}},
+			{"\u035F", [][]int{{0, 233}}},
+			{"\u035D", [][]int{{0, 234}}},
+			{"\u035E", [][]int{{0, 234}}},
+			{"\u0345", [][]int{{0, 240}}},
+
+			// starters
+			{"a", [][]int{{100}}},
+			{"b", [][]int{{200}}},
+			{"c", [][]int{{300}}},
+			{"\u03B1", [][]int{{900}}},
+			{"\x01", [][]int{{0, 0, 0, 0}}},
+
+			// contractions
+			{"a\u0300", [][]int{{101}}},
+			{"a\u0301", [][]int{{102}}},
+			{"a\u035E", [][]int{{110}}},
+			{"a\u035Eb\u035E", [][]int{{115}}},
+			{"ac\u035Eaca\u035E", [][]int{{116}}},
+			{"a\u035Db\u035D", [][]int{{117}}},
+			{"a\u0301\u035Db", [][]int{{120}}},
+			{"a\u0301\u035F", [][]int{{121}}},
+			{"a\u0301\u035Fb", [][]int{{119}}},
+			{"\u03B1\u0345", [][]int{{901}, {902}}},
+			{"\u302E\u302F", [][]int{{0, 131}, {0, 131}}},
+			{"\u302F\u18A9", [][]int{{0, 130}}},
+		}...),
+		[]check{
+			{"a\x01\u0300", 1, ColElems{w(100)}},
+			{"ab", 1, ColElems{w(100)}},                              // closing segment
+			{"a\u0316\u0300b", 5, ColElems{w(101), w(0, 220)}},       // closing segment
+			{"a\u0316\u0300", 5, ColElems{w(101), w(0, 220)}},        // no closing segment
+			{"a\u0316\u0300\u035Cb", 5, ColElems{w(101), w(0, 220)}}, // completes before segment end
+			{"a\u0316\u0300\u035C", 5, ColElems{w(101), w(0, 220)}},  // completes before segment end
+
+			{"a\u0316\u0301b", 5, ColElems{w(102), w(0, 220)}},       // closing segment
+			{"a\u0316\u0301", 5, ColElems{w(102), w(0, 220)}},        // no closing segment
+			{"a\u0316\u0301\u035Cb", 5, ColElems{w(102), w(0, 220)}}, // completes before segment end
+			{"a\u0316\u0301\u035C", 5, ColElems{w(102), w(0, 220)}},  // completes before segment end
+
+			// match blocked by modifier with same ccc
+			{"a\u0301\u0315\u031A\u035Fb", 3, ColElems{w(102)}},
+
+			// multiple gaps
+			{"a\u0301\u035Db", 6, ColElems{w(120)}},
+			{"a\u0301\u035F", 5, ColElems{w(121)}},
+			{"a\u0301\u035Fb", 6, ColElems{w(119)}},
+			{"a\u0316\u0301\u035F", 7, ColElems{w(121), w(0, 220)}},
+			{"a\u0301\u0315\u035Fb", 7, ColElems{w(121), w(0, 232)}},
+			{"a\u0316\u0301\u0315\u035Db", 5, ColElems{w(102), w(0, 220)}},
+			{"a\u0316\u0301\u0315\u035F", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
+			{"a\u0316\u0301\u0315\u035Fb", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
+			{"a\u0316\u0301\u0315\u035F\u035D", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
+			{"a\u0316\u0301\u0315\u035F\u035Db", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
+
+			// handling of segment overflow
+			{ // just fits within segment
+				"a" + string(modSeq[:30]) + "\u0301",
+				3 + len(string(modSeq[:30])),
+				append(ColElems{w(102)}, modW[:30]...),
+			},
+			{"a" + string(modSeq[:31]) + "\u0301", 1, ColElems{w(100)}}, // overflow
+			{"a" + string(modSeq) + "\u0301", 1, ColElems{w(100)}},
+			{ // just fits within segment with two interstitial runes
+				"a" + string(modSeq[:28]) + "\u0301\u0315\u035F",
+				7 + len(string(modSeq[:28])),
+				append(append(ColElems{w(121)}, modW[:28]...), w(0, 232)),
+			},
+			{ // second half does not fit within segment
+				"a" + string(modSeq[:29]) + "\u0301\u0315\u035F",
+				3 + len(string(modSeq[:29])),
+				append(ColElems{w(102)}, modW[:29]...),
+			},
+
+			// discontinuity can only occur in last normalization segment
+			{"a\u035Eb\u035E", 6, ColElems{w(115)}},
+			{"a\u0316\u035Eb\u035E", 5, ColElems{w(110), w(0, 220)}},
+			{"a\u035Db\u035D", 6, ColElems{w(117)}},
+			{"a\u0316\u035Db\u035D", 1, ColElems{w(100)}},
+			{"a\u035Eb\u0316\u035E", 8, ColElems{w(115), w(0, 220)}},
+			{"a\u035Db\u0316\u035D", 8, ColElems{w(117), w(0, 220)}},
+			{"ac\u035Eaca\u035E", 9, ColElems{w(116)}},
+			{"a\u0316c\u035Eaca\u035E", 1, ColElems{w(100)}},
+			{"ac\u035Eac\u0316a\u035E", 1, ColElems{w(100)}},
+
+			// expanding contraction
+			{"\u03B1\u0345", 4, ColElems{w(901), w(902)}},
+
+			// Theoretical possibilities
+			// contraction within a gap
+			{"a\u302F\u18A9\u0301", 9, ColElems{w(102), w(0, 130)}},
+			// expansion within a gap
+			{"a\u0317\u0301", 5, ColElems{w(102), w(0, 220), w(0, 220)}},
+			// repeating CCC blocks last modifier
+			{"a\u302E\u302F\u0301", 1, ColElems{w(100)}},
+			// The trailing combining characters (with lower CCC) should block the first one.
+			// TODO: make the following pass.
+			// {"a\u035E\u0316\u0316", 1, ColElems{w(100)}},
+			{"a\u035F\u035Eb", 5, ColElems{w(110), w(0, 233)}},
+			// Last combiner should match after normalization.
+			// TODO: make the following pass.
+			// {"a\u035D\u0301", 3, ColElems{w(102), w(0, 234)}},
+			// The first combiner is blocking the second one as they have the same CCC.
+			{"a\u035D\u035Eb", 1, ColElems{w(100)}},
+		},
+	},
+}
+
+func TestAppendNext(t *testing.T) {
+	for i, tt := range appendNextTests {
+		c, err := makeTable(tt.in)
+		if err != nil {
+			t.Errorf("%d: error creating table: %v", i, err)
+			continue
+		}
+		for j, chk := range tt.chk {
+			ws, n := c.t.AppendNext(nil, []byte(chk.in))
+			if n != chk.n {
+				t.Errorf("%d:%d: bytes consumed was %d; want %d", i, j, n, chk.n)
+			}
+			out := convertFromWeights(chk.out)
+			if len(ws) != len(out) {
+				t.Errorf("%d:%d: len(ws) was %d; want %d (%X vs %X)\n%X", i, j, len(ws), len(out), ws, out, chk.in)
+				continue
+			}
+			for k, w := range ws {
+				w, _ = colltab.MakeElem(w.Primary(), w.Secondary(), int(w.Tertiary()), 0)
+				if w != out[k] {
+					t.Errorf("%d:%d: Weights %d was %X; want %X", i, j, k, w, out[k])
+				}
+			}
+		}
+	}
+}

vendor/golang.org/x/text/collate/tools/colcmp/Makefile

@@ -0,0 +1,7 @@
+# Copyright 2012 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+chars:
+	go run ../../maketables.go -tables=chars -package=main > chars.go
+	gofmt -w -s chars.go

vendor/golang.org/x/text/collate/tools/colcmp/col.go

@@ -0,0 +1,97 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"log"
+	"unicode/utf16"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+)
+
+// Input holds an input string in both UTF-8 and UTF-16 format.
+type Input struct {
+	index int // used for restoring to original random order
+	UTF8  []byte
+	UTF16 []uint16
+	key   []byte // used for sorting
+}
+
+func (i Input) String() string {
+	return string(i.UTF8)
+}
+
+func makeInput(s8 []byte, s16 []uint16) Input {
+	return Input{UTF8: s8, UTF16: s16}
+}
+
+func makeInputString(s string) Input {
+	return Input{
+		UTF8:  []byte(s),
+		UTF16: utf16.Encode([]rune(s)),
+	}
+}
+
+// Collator is an interface for architecture-specific implementations of collation.
+type Collator interface {
+	// Key generates a sort key for the given input.  Implemenations
+	// may return nil if a collator does not support sort keys.
+	Key(s Input) []byte
+
+	// Compare returns -1 if a < b, 1 if a > b and 0 if a == b.
+	Compare(a, b Input) int
+}
+
+// CollatorFactory creates a Collator for a given language tag.
+type CollatorFactory struct {
+	name        string
+	makeFn      func(tag string) (Collator, error)
+	description string
+}
+
+var collators = []CollatorFactory{}
+
+// AddFactory registers f as a factory for an implementation of Collator.
+func AddFactory(f CollatorFactory) {
+	collators = append(collators, f)
+}
+
+func getCollator(name, locale string) Collator {
+	for _, f := range collators {
+		if f.name == name {
+			col, err := f.makeFn(locale)
+			if err != nil {
+				log.Fatal(err)
+			}
+			return col
+		}
+	}
+	log.Fatalf("collator of type %q not found", name)
+	return nil
+}
+
+// goCollator is an implemention of Collator using go's own collator.
+type goCollator struct {
+	c   *collate.Collator
+	buf collate.Buffer
+}
+
+func init() {
+	AddFactory(CollatorFactory{"go", newGoCollator, "Go's native collator implementation."})
+}
+
+func newGoCollator(loc string) (Collator, error) {
+	c := &goCollator{c: collate.New(language.Make(loc))}
+	return c, nil
+}
+
+func (c *goCollator) Key(b Input) []byte {
+	return c.c.Key(&c.buf, b.UTF8)
+}
+
+func (c *goCollator) Compare(a, b Input) int {
+	return c.c.Compare(a.UTF8, b.UTF8)
+}

vendor/golang.org/x/text/collate/tools/colcmp/colcmp.go

@@ -0,0 +1,529 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main // import "golang.org/x/text/collate/tools/colcmp"
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"runtime/pprof"
+	"sort"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+var (
+	doNorm  = flag.Bool("norm", false, "normalize input strings")
+	cases   = flag.Bool("case", false, "generate case variants")
+	verbose = flag.Bool("verbose", false, "print results")
+	debug   = flag.Bool("debug", false, "output debug information")
+	locales = flag.String("locale", "en_US", "the locale to use. May be a comma-separated list for some commands.")
+	col     = flag.String("col", "go", "collator to test")
+	gold    = flag.String("gold", "go", "collator used as the gold standard")
+	usecmp  = flag.Bool("usecmp", false,
+		`use comparison instead of sort keys when sorting.  Must be "test", "gold" or "both"`)
+	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+	exclude    = flag.String("exclude", "", "exclude errors that contain any of the characters")
+	limit      = flag.Int("limit", 5000000, "maximum number of samples to generate for one run")
+)
+
+func failOnError(err error) {
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+// Test holds test data for testing a locale-collator pair.
+// Test also provides functionality that is commonly used by the various commands.
+type Test struct {
+	ctxt    *Context
+	Name    string
+	Locale  string
+	ColName string
+
+	Col        Collator
+	UseCompare bool
+
+	Input    []Input
+	Duration time.Duration
+
+	start time.Time
+	msg   string
+	count int
+}
+
+func (t *Test) clear() {
+	t.Col = nil
+	t.Input = nil
+}
+
+const (
+	msgGeneratingInput = "generating input"
+	msgGeneratingKeys  = "generating keys"
+	msgSorting         = "sorting"
+)
+
+var lastLen = 0
+
+func (t *Test) SetStatus(msg string) {
+	if *debug || *verbose {
+		fmt.Printf("%s: %s...\n", t.Name, msg)
+	} else if t.ctxt.out != nil {
+		fmt.Fprint(t.ctxt.out, strings.Repeat(" ", lastLen))
+		fmt.Fprint(t.ctxt.out, strings.Repeat("\b", lastLen))
+		fmt.Fprint(t.ctxt.out, msg, "...")
+		lastLen = len(msg) + 3
+		fmt.Fprint(t.ctxt.out, strings.Repeat("\b", lastLen))
+	}
+}
+
+// Start is used by commands to signal the start of an operation.
+func (t *Test) Start(msg string) {
+	t.SetStatus(msg)
+	t.count = 0
+	t.msg = msg
+	t.start = time.Now()
+}
+
+// Stop is used by commands to signal the end of an operation.
+func (t *Test) Stop() (time.Duration, int) {
+	d := time.Now().Sub(t.start)
+	t.Duration += d
+	if *debug || *verbose {
+		fmt.Printf("%s: %s done. (%.3fs /%dK ops)\n", t.Name, t.msg, d.Seconds(), t.count/1000)
+	}
+	return d, t.count
+}
+
+// generateKeys generates sort keys for all the inputs.
+func (t *Test) generateKeys() {
+	for i, s := range t.Input {
+		b := t.Col.Key(s)
+		t.Input[i].key = b
+		if *debug {
+			fmt.Printf("%s (%X): %X\n", string(s.UTF8), s.UTF16, b)
+		}
+	}
+}
+
+// Sort sorts the inputs. It generates sort keys if this is required by the
+// chosen sort method.
+func (t *Test) Sort() (tkey, tsort time.Duration, nkey, nsort int) {
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		failOnError(err)
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+	if t.UseCompare || t.Col.Key(t.Input[0]) == nil {
+		t.Start(msgSorting)
+		sort.Sort(&testCompare{*t})
+		tsort, nsort = t.Stop()
+	} else {
+		t.Start(msgGeneratingKeys)
+		t.generateKeys()
+		t.count = len(t.Input)
+		tkey, nkey = t.Stop()
+		t.Start(msgSorting)
+		sort.Sort(t)
+		tsort, nsort = t.Stop()
+	}
+	return
+}
+
+func (t *Test) Swap(a, b int) {
+	t.Input[a], t.Input[b] = t.Input[b], t.Input[a]
+}
+
+func (t *Test) Less(a, b int) bool {
+	t.count++
+	return bytes.Compare(t.Input[a].key, t.Input[b].key) == -1
+}
+
+func (t Test) Len() int {
+	return len(t.Input)
+}
+
+type testCompare struct {
+	Test
+}
+
+func (t *testCompare) Less(a, b int) bool {
+	t.count++
+	return t.Col.Compare(t.Input[a], t.Input[b]) == -1
+}
+
+type testRestore struct {
+	Test
+}
+
+func (t *testRestore) Less(a, b int) bool {
+	return t.Input[a].index < t.Input[b].index
+}
+
+// GenerateInput generates input phrases for the locale tested by t.
+func (t *Test) GenerateInput() {
+	t.Input = nil
+	if t.ctxt.lastLocale != t.Locale {
+		gen := phraseGenerator{}
+		gen.init(t.Locale)
+		t.SetStatus(msgGeneratingInput)
+		t.ctxt.lastInput = nil // allow the previous value to be garbage collected.
+		t.Input = gen.generate(*doNorm)
+		t.ctxt.lastInput = t.Input
+		t.ctxt.lastLocale = t.Locale
+	} else {
+		t.Input = t.ctxt.lastInput
+		for i := range t.Input {
+			t.Input[i].key = nil
+		}
+		sort.Sort(&testRestore{*t})
+	}
+}
+
+// Context holds all tests and settings translated from command line options.
+type Context struct {
+	test []*Test
+	last *Test
+
+	lastLocale string
+	lastInput  []Input
+
+	out io.Writer
+}
+
+func (ts *Context) Printf(format string, a ...interface{}) {
+	ts.assertBuf()
+	fmt.Fprintf(ts.out, format, a...)
+}
+
+func (ts *Context) Print(a ...interface{}) {
+	ts.assertBuf()
+	fmt.Fprint(ts.out, a...)
+}
+
+// assertBuf sets up an io.Writer for ouput, if it doesn't already exist.
+// In debug and verbose mode, output is buffered so that the regular output
+// will not interfere with the additional output.  Otherwise, output is
+// written directly to stdout for a more responsive feel.
+func (ts *Context) assertBuf() {
+	if ts.out != nil {
+		return
+	}
+	if *debug || *verbose {
+		ts.out = &bytes.Buffer{}
+	} else {
+		ts.out = os.Stdout
+	}
+}
+
+// flush flushes the contents of ts.out to stdout, if it is not stdout already.
+func (ts *Context) flush() {
+	if ts.out != nil {
+		if _, ok := ts.out.(io.ReadCloser); !ok {
+			io.Copy(os.Stdout, ts.out.(io.Reader))
+		}
+	}
+}
+
+// parseTests creates all tests from command lines and returns
+// a Context to hold them.
+func parseTests() *Context {
+	ctxt := &Context{}
+	colls := strings.Split(*col, ",")
+	for _, loc := range strings.Split(*locales, ",") {
+		loc = strings.TrimSpace(loc)
+		for _, name := range colls {
+			name = strings.TrimSpace(name)
+			col := getCollator(name, loc)
+			ctxt.test = append(ctxt.test, &Test{
+				ctxt:       ctxt,
+				Locale:     loc,
+				ColName:    name,
+				UseCompare: *usecmp,
+				Col:        col,
+			})
+		}
+	}
+	return ctxt
+}
+
+func (c *Context) Len() int {
+	return len(c.test)
+}
+
+func (c *Context) Test(i int) *Test {
+	if c.last != nil {
+		c.last.clear()
+	}
+	c.last = c.test[i]
+	return c.last
+}
+
+func parseInput(args []string) []Input {
+	input := []Input{}
+	for _, s := range args {
+		rs := []rune{}
+		for len(s) > 0 {
+			var r rune
+			r, _, s, _ = strconv.UnquoteChar(s, '\'')
+			rs = append(rs, r)
+		}
+		s = string(rs)
+		if *doNorm {
+			s = norm.NFD.String(s)
+		}
+		input = append(input, makeInputString(s))
+	}
+	return input
+}
+
+// A Command is an implementation of a colcmp command.
+type Command struct {
+	Run   func(cmd *Context, args []string)
+	Usage string
+	Short string
+	Long  string
+}
+
+func (cmd Command) Name() string {
+	return strings.SplitN(cmd.Usage, " ", 2)[0]
+}
+
+var commands = []*Command{
+	cmdSort,
+	cmdBench,
+	cmdRegress,
+}
+
+const sortHelp = `
+Sort sorts a given list of strings.  Strings are separated by whitespace.
+`
+
+var cmdSort = &Command{
+	Run:   runSort,
+	Usage: "sort <string>*",
+	Short: "sort a given list of strings",
+	Long:  sortHelp,
+}
+
+func runSort(ctxt *Context, args []string) {
+	input := parseInput(args)
+	if len(input) == 0 {
+		log.Fatalf("Nothing to sort.")
+	}
+	if ctxt.Len() > 1 {
+		ctxt.Print("COLL  LOCALE RESULT\n")
+	}
+	for i := 0; i < ctxt.Len(); i++ {
+		t := ctxt.Test(i)
+		t.Input = append(t.Input, input...)
+		t.Sort()
+		if ctxt.Len() > 1 {
+			ctxt.Printf("%-5s %-5s  ", t.ColName, t.Locale)
+		}
+		for _, s := range t.Input {
+			ctxt.Print(string(s.UTF8), " ")
+		}
+		ctxt.Print("\n")
+	}
+}
+
+const benchHelp = `
+Bench runs a benchmark for the given list of collator implementations.
+If no collator implementations are given, the go collator will be used.
+`
+
+var cmdBench = &Command{
+	Run:   runBench,
+	Usage: "bench",
+	Short: "benchmark a given list of collator implementations",
+	Long:  benchHelp,
+}
+
+func runBench(ctxt *Context, args []string) {
+	ctxt.Printf("%-7s %-5s %-6s %-24s %-24s %-5s %s\n", "LOCALE", "COLL", "N", "KEYS", "SORT", "AVGLN", "TOTAL")
+	for i := 0; i < ctxt.Len(); i++ {
+		t := ctxt.Test(i)
+		ctxt.Printf("%-7s %-5s ", t.Locale, t.ColName)
+		t.GenerateInput()
+		ctxt.Printf("%-6s ", fmt.Sprintf("%dK", t.Len()/1000))
+		tkey, tsort, nkey, nsort := t.Sort()
+		p := func(dur time.Duration, n int) {
+			s := ""
+			if dur > 0 {
+				s = fmt.Sprintf("%6.3fs ", dur.Seconds())
+				if n > 0 {
+					s += fmt.Sprintf("%15s", fmt.Sprintf("(%4.2f ns/op)", float64(dur)/float64(n)))
+				}
+			}
+			ctxt.Printf("%-24s ", s)
+		}
+		p(tkey, nkey)
+		p(tsort, nsort)
+
+		total := 0
+		for _, s := range t.Input {
+			total += len(s.key)
+		}
+		ctxt.Printf("%-5d ", total/t.Len())
+		ctxt.Printf("%6.3fs\n", t.Duration.Seconds())
+		if *debug {
+			for _, s := range t.Input {
+				fmt.Print(string(s.UTF8), " ")
+			}
+			fmt.Println()
+		}
+	}
+}
+
+const regressHelp = `
+Regress runs a monkey test by comparing the results of randomly generated tests
+between two implementations of a collator. The user may optionally pass a list
+of strings to regress against instead of the default test set.
+`
+
+var cmdRegress = &Command{
+	Run:   runRegress,
+	Usage: "regress -gold=<col> -test=<col> [string]*",
+	Short: "run a monkey test between two collators",
+	Long:  regressHelp,
+}
+
+const failedKeyCompare = `
+%s:%d: incorrect comparison result for input:
+    a:   %q (%.4X)
+    key: %s
+    b:   %q (%.4X)
+    key: %s
+    Compare(a, b) = %d; want %d.
+
+  gold keys:
+	a:   %s
+	b:   %s
+`
+
+const failedCompare = `
+%s:%d: incorrect comparison result for input:
+    a:   %q (%.4X)
+    b:   %q (%.4X)
+    Compare(a, b) = %d; want %d.
+`
+
+func keyStr(b []byte) string {
+	buf := &bytes.Buffer{}
+	for _, v := range b {
+		fmt.Fprintf(buf, "%.2X ", v)
+	}
+	return buf.String()
+}
+
+func runRegress(ctxt *Context, args []string) {
+	input := parseInput(args)
+	for i := 0; i < ctxt.Len(); i++ {
+		t := ctxt.Test(i)
+		if len(input) > 0 {
+			t.Input = append(t.Input, input...)
+		} else {
+			t.GenerateInput()
+		}
+		t.Sort()
+		count := 0
+		gold := getCollator(*gold, t.Locale)
+		for i := 1; i < len(t.Input); i++ {
+			ia := t.Input[i-1]
+			ib := t.Input[i]
+			if bytes.IndexAny(ib.UTF8, *exclude) != -1 {
+				i++
+				continue
+			}
+			if bytes.IndexAny(ia.UTF8, *exclude) != -1 {
+				continue
+			}
+			goldCmp := gold.Compare(ia, ib)
+			if cmp := bytes.Compare(ia.key, ib.key); cmp != goldCmp {
+				count++
+				a := string(ia.UTF8)
+				b := string(ib.UTF8)
+				fmt.Printf(failedKeyCompare, t.Locale, i-1, a, []rune(a), keyStr(ia.key), b, []rune(b), keyStr(ib.key), cmp, goldCmp, keyStr(gold.Key(ia)), keyStr(gold.Key(ib)))
+			} else if cmp := t.Col.Compare(ia, ib); cmp != goldCmp {
+				count++
+				a := string(ia.UTF8)
+				b := string(ib.UTF8)
+				fmt.Printf(failedCompare, t.Locale, i-1, a, []rune(a), b, []rune(b), cmp, goldCmp)
+			}
+		}
+		if count > 0 {
+			ctxt.Printf("Found %d inconsistencies in %d entries.\n", count, t.Len()-1)
+		}
+	}
+}
+
+const helpTemplate = `
+colcmp is a tool for testing and benchmarking collation
+
+Usage: colcmp command [arguments]
+
+The commands are:
+{{range .}}
+    {{.Name | printf "%-11s"}} {{.Short}}{{end}}
+
+Use "col help [topic]" for more information about that topic.
+`
+
+const detailedHelpTemplate = `
+Usage: colcmp {{.Usage}}
+
+{{.Long | trim}}
+`
+
+func runHelp(args []string) {
+	t := template.New("help")
+	t.Funcs(template.FuncMap{"trim": strings.TrimSpace})
+	if len(args) < 1 {
+		template.Must(t.Parse(helpTemplate))
+		failOnError(t.Execute(os.Stderr, &commands))
+	} else {
+		for _, cmd := range commands {
+			if cmd.Name() == args[0] {
+				template.Must(t.Parse(detailedHelpTemplate))
+				failOnError(t.Execute(os.Stderr, cmd))
+				os.Exit(0)
+			}
+		}
+		log.Fatalf("Unknown command %q. Run 'colcmp help'.", args[0])
+	}
+	os.Exit(0)
+}
+
+func main() {
+	flag.Parse()
+	log.SetFlags(0)
+
+	ctxt := parseTests()
+
+	if flag.NArg() < 1 {
+		runHelp(nil)
+	}
+	args := flag.Args()[1:]
+	if flag.Arg(0) == "help" {
+		runHelp(args)
+	}
+	for _, cmd := range commands {
+		if cmd.Name() == flag.Arg(0) {
+			cmd.Run(ctxt, args)
+			ctxt.flush()
+			return
+		}
+	}
+	runHelp(flag.Args())
+}

vendor/golang.org/x/text/collate/tools/colcmp/darwin.go

@@ -0,0 +1,111 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin
+
+package main
+
+/*
+#cgo LDFLAGS: -framework CoreFoundation
+#include <CoreFoundation/CFBase.h>
+#include <CoreFoundation/CoreFoundation.h>
+*/
+import "C"
+import (
+	"unsafe"
+)
+
+func init() {
+	AddFactory(CollatorFactory{"osx", newOSX16Collator,
+		"OS X/Darwin collator, using native strings."})
+	AddFactory(CollatorFactory{"osx8", newOSX8Collator,
+		"OS X/Darwin collator for UTF-8."})
+}
+
+func osxUInt8P(s []byte) *C.UInt8 {
+	return (*C.UInt8)(unsafe.Pointer(&s[0]))
+}
+
+func osxCharP(s []uint16) *C.UniChar {
+	return (*C.UniChar)(unsafe.Pointer(&s[0]))
+}
+
+// osxCollator implements an Collator based on OS X's CoreFoundation.
+type osxCollator struct {
+	loc C.CFLocaleRef
+	opt C.CFStringCompareFlags
+}
+
+func (c *osxCollator) init(locale string) {
+	l := C.CFStringCreateWithBytes(
+		nil,
+		osxUInt8P([]byte(locale)),
+		C.CFIndex(len(locale)),
+		C.kCFStringEncodingUTF8,
+		C.Boolean(0),
+	)
+	c.loc = C.CFLocaleCreate(nil, l)
+}
+
+func newOSX8Collator(locale string) (Collator, error) {
+	c := &osx8Collator{}
+	c.init(locale)
+	return c, nil
+}
+
+func newOSX16Collator(locale string) (Collator, error) {
+	c := &osx16Collator{}
+	c.init(locale)
+	return c, nil
+}
+
+func (c osxCollator) Key(s Input) []byte {
+	return nil // sort keys not supported by OS X CoreFoundation
+}
+
+type osx8Collator struct {
+	osxCollator
+}
+
+type osx16Collator struct {
+	osxCollator
+}
+
+func (c osx16Collator) Compare(a, b Input) int {
+	sa := C.CFStringCreateWithCharactersNoCopy(
+		nil,
+		osxCharP(a.UTF16),
+		C.CFIndex(len(a.UTF16)),
+		nil,
+	)
+	sb := C.CFStringCreateWithCharactersNoCopy(
+		nil,
+		osxCharP(b.UTF16),
+		C.CFIndex(len(b.UTF16)),
+		nil,
+	)
+	_range := C.CFRangeMake(0, C.CFStringGetLength(sa))
+	return int(C.CFStringCompareWithOptionsAndLocale(sa, sb, _range, c.opt, c.loc))
+}
+
+func (c osx8Collator) Compare(a, b Input) int {
+	sa := C.CFStringCreateWithBytesNoCopy(
+		nil,
+		osxUInt8P(a.UTF8),
+		C.CFIndex(len(a.UTF8)),
+		C.kCFStringEncodingUTF8,
+		C.Boolean(0),
+		nil,
+	)
+	sb := C.CFStringCreateWithBytesNoCopy(
+		nil,
+		osxUInt8P(b.UTF8),
+		C.CFIndex(len(b.UTF8)),
+		C.kCFStringEncodingUTF8,
+		C.Boolean(0),
+		nil,
+	)
+	_range := C.CFRangeMake(0, C.CFStringGetLength(sa))
+	return int(C.CFStringCompareWithOptionsAndLocale(sa, sb, _range, c.opt, c.loc))
+}

vendor/golang.org/x/text/collate/tools/colcmp/gen.go

@@ -0,0 +1,183 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"math"
+	"math/rand"
+	"strings"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+// TODO: replace with functionality in language package.
+// parent computes the parent language for the given language.
+// It returns false if the parent is already root.
+func parent(locale string) (parent string, ok bool) {
+	if locale == "und" {
+		return "", false
+	}
+	if i := strings.LastIndex(locale, "-"); i != -1 {
+		return locale[:i], true
+	}
+	return "und", true
+}
+
+// rewriter is used to both unique strings and create variants of strings
+// to add to the test set.
+type rewriter struct {
+	seen     map[string]bool
+	addCases bool
+}
+
+func newRewriter() *rewriter {
+	return &rewriter{
+		seen: make(map[string]bool),
+	}
+}
+
+func (r *rewriter) insert(a []string, s string) []string {
+	if !r.seen[s] {
+		r.seen[s] = true
+		a = append(a, s)
+	}
+	return a
+}
+
+// rewrite takes a sequence of strings in, adds variants of the these strings
+// based on options and removes duplicates.
+func (r *rewriter) rewrite(ss []string) []string {
+	ns := []string{}
+	for _, s := range ss {
+		ns = r.insert(ns, s)
+		if r.addCases {
+			rs := []rune(s)
+			rn := rs[0]
+			for c := unicode.SimpleFold(rn); c != rn; c = unicode.SimpleFold(c) {
+				rs[0] = c
+				ns = r.insert(ns, string(rs))
+			}
+		}
+	}
+	return ns
+}
+
+// exemplarySet holds a parsed set of characters from the exemplarCharacters table.
+type exemplarySet struct {
+	typ       exemplarType
+	set       []string
+	charIndex int // cumulative total of phrases, including this set
+}
+
+type phraseGenerator struct {
+	sets [exN]exemplarySet
+	n    int
+}
+
+func (g *phraseGenerator) init(id string) {
+	ec := exemplarCharacters
+	loc := language.Make(id).String()
+	// get sets for locale or parent locale if the set is not defined.
+	for i := range g.sets {
+		for p, ok := loc, true; ok; p, ok = parent(p) {
+			if set, ok := ec[p]; ok && set[i] != "" {
+				g.sets[i].set = strings.Split(set[i], " ")
+				break
+			}
+		}
+	}
+	r := newRewriter()
+	r.addCases = *cases
+	for i := range g.sets {
+		g.sets[i].set = r.rewrite(g.sets[i].set)
+	}
+	// compute indexes
+	for i, set := range g.sets {
+		g.n += len(set.set)
+		g.sets[i].charIndex = g.n
+	}
+}
+
+// phrase returns the ith phrase, where i < g.n.
+func (g *phraseGenerator) phrase(i int) string {
+	for _, set := range g.sets {
+		if i < set.charIndex {
+			return set.set[i-(set.charIndex-len(set.set))]
+		}
+	}
+	panic("index out of range")
+}
+
+// generate generates inputs by combining all pairs of examplar strings.
+// If doNorm is true, all input strings are normalized to NFC.
+// TODO: allow other variations, statistical models, and random
+// trailing sequences.
+func (g *phraseGenerator) generate(doNorm bool) []Input {
+	const (
+		M         = 1024 * 1024
+		buf8Size  = 30 * M
+		buf16Size = 10 * M
+	)
+	// TODO: use a better way to limit the input size.
+	if sq := int(math.Sqrt(float64(*limit))); g.n > sq {
+		g.n = sq
+	}
+	size := g.n * g.n
+	a := make([]Input, 0, size)
+	buf8 := make([]byte, 0, buf8Size)
+	buf16 := make([]uint16, 0, buf16Size)
+
+	addInput := func(str string) {
+		buf8 = buf8[len(buf8):]
+		buf16 = buf16[len(buf16):]
+		if len(str) > cap(buf8) {
+			buf8 = make([]byte, 0, buf8Size)
+		}
+		if len(str) > cap(buf16) {
+			buf16 = make([]uint16, 0, buf16Size)
+		}
+		if doNorm {
+			buf8 = norm.NFD.AppendString(buf8, str)
+		} else {
+			buf8 = append(buf8, str...)
+		}
+		buf16 = appendUTF16(buf16, buf8)
+		a = append(a, makeInput(buf8, buf16))
+	}
+	for i := 0; i < g.n; i++ {
+		p1 := g.phrase(i)
+		addInput(p1)
+		for j := 0; j < g.n; j++ {
+			p2 := g.phrase(j)
+			addInput(p1 + p2)
+		}
+	}
+	// permutate
+	rnd := rand.New(rand.NewSource(int64(rand.Int())))
+	for i := range a {
+		j := i + rnd.Intn(len(a)-i)
+		a[i], a[j] = a[j], a[i]
+		a[i].index = i // allow restoring this order if input is used multiple times.
+	}
+	return a
+}
+
+func appendUTF16(buf []uint16, s []byte) []uint16 {
+	for len(s) > 0 {
+		r, sz := utf8.DecodeRune(s)
+		s = s[sz:]
+		r1, r2 := utf16.EncodeRune(r)
+		if r1 != 0xFFFD {
+			buf = append(buf, uint16(r1), uint16(r2))
+		} else {
+			buf = append(buf, uint16(r))
+		}
+	}
+	return buf
+}

vendor/golang.org/x/text/collate/tools/colcmp/icu.go

@@ -0,0 +1,209 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build icu
+
+package main
+
+/*
+#cgo LDFLAGS: -licui18n -licuuc
+#include <stdlib.h>
+#include <unicode/ucol.h>
+#include <unicode/uiter.h>
+#include <unicode/utypes.h>
+*/
+import "C"
+import (
+	"fmt"
+	"log"
+	"unicode/utf16"
+	"unicode/utf8"
+	"unsafe"
+)
+
+func init() {
+	AddFactory(CollatorFactory{"icu", newUTF16,
+		"Main ICU collator, using native strings."})
+	AddFactory(CollatorFactory{"icu8", newUTF8iter,
+		"ICU collator using ICU iterators to process UTF8."})
+	AddFactory(CollatorFactory{"icu16", newUTF8conv,
+		"ICU collation by first converting UTF8 to UTF16."})
+}
+
+func icuCharP(s []byte) *C.char {
+	return (*C.char)(unsafe.Pointer(&s[0]))
+}
+
+func icuUInt8P(s []byte) *C.uint8_t {
+	return (*C.uint8_t)(unsafe.Pointer(&s[0]))
+}
+
+func icuUCharP(s []uint16) *C.UChar {
+	return (*C.UChar)(unsafe.Pointer(&s[0]))
+}
+func icuULen(s []uint16) C.int32_t {
+	return C.int32_t(len(s))
+}
+func icuSLen(s []byte) C.int32_t {
+	return C.int32_t(len(s))
+}
+
+// icuCollator implements a Collator based on ICU.
+type icuCollator struct {
+	loc    *C.char
+	col    *C.UCollator
+	keyBuf []byte
+}
+
+const growBufSize = 10 * 1024 * 1024
+
+func (c *icuCollator) init(locale string) error {
+	err := C.UErrorCode(0)
+	c.loc = C.CString(locale)
+	c.col = C.ucol_open(c.loc, &err)
+	if err > 0 {
+		return fmt.Errorf("failed opening collator for %q", locale)
+	} else if err < 0 {
+		loc := C.ucol_getLocaleByType(c.col, 0, &err)
+		fmt, ok := map[int]string{
+			-127: "warning: using default collator: %s",
+			-128: "warning: using fallback collator: %s",
+		}[int(err)]
+		if ok {
+			log.Printf(fmt, C.GoString(loc))
+		}
+	}
+	c.keyBuf = make([]byte, 0, growBufSize)
+	return nil
+}
+
+func (c *icuCollator) buf() (*C.uint8_t, C.int32_t) {
+	if len(c.keyBuf) == cap(c.keyBuf) {
+		c.keyBuf = make([]byte, 0, growBufSize)
+	}
+	b := c.keyBuf[len(c.keyBuf):cap(c.keyBuf)]
+	return icuUInt8P(b), icuSLen(b)
+}
+
+func (c *icuCollator) extendBuf(n C.int32_t) []byte {
+	end := len(c.keyBuf) + int(n)
+	if end > cap(c.keyBuf) {
+		if len(c.keyBuf) == 0 {
+			log.Fatalf("icuCollator: max string size exceeded: %v > %v", n, growBufSize)
+		}
+		c.keyBuf = make([]byte, 0, growBufSize)
+		return nil
+	}
+	b := c.keyBuf[len(c.keyBuf):end]
+	c.keyBuf = c.keyBuf[:end]
+	return b
+}
+
+func (c *icuCollator) Close() error {
+	C.ucol_close(c.col)
+	C.free(unsafe.Pointer(c.loc))
+	return nil
+}
+
+// icuUTF16 implements the Collator interface.
+type icuUTF16 struct {
+	icuCollator
+}
+
+func newUTF16(locale string) (Collator, error) {
+	c := &icuUTF16{}
+	return c, c.init(locale)
+}
+
+func (c *icuUTF16) Compare(a, b Input) int {
+	return int(C.ucol_strcoll(c.col, icuUCharP(a.UTF16), icuULen(a.UTF16), icuUCharP(b.UTF16), icuULen(b.UTF16)))
+}
+
+func (c *icuUTF16) Key(s Input) []byte {
+	bp, bn := c.buf()
+	n := C.ucol_getSortKey(c.col, icuUCharP(s.UTF16), icuULen(s.UTF16), bp, bn)
+	if b := c.extendBuf(n); b != nil {
+		return b
+	}
+	return c.Key(s)
+}
+
+// icuUTF8iter implements the Collator interface
+// This implementation wraps the UTF8 string in an iterator
+// which is passed to the collator.
+type icuUTF8iter struct {
+	icuCollator
+	a, b C.UCharIterator
+}
+
+func newUTF8iter(locale string) (Collator, error) {
+	c := &icuUTF8iter{}
+	return c, c.init(locale)
+}
+
+func (c *icuUTF8iter) Compare(a, b Input) int {
+	err := C.UErrorCode(0)
+	C.uiter_setUTF8(&c.a, icuCharP(a.UTF8), icuSLen(a.UTF8))
+	C.uiter_setUTF8(&c.b, icuCharP(b.UTF8), icuSLen(b.UTF8))
+	return int(C.ucol_strcollIter(c.col, &c.a, &c.b, &err))
+}
+
+func (c *icuUTF8iter) Key(s Input) []byte {
+	err := C.UErrorCode(0)
+	state := [2]C.uint32_t{}
+	C.uiter_setUTF8(&c.a, icuCharP(s.UTF8), icuSLen(s.UTF8))
+	bp, bn := c.buf()
+	n := C.ucol_nextSortKeyPart(c.col, &c.a, &(state[0]), bp, bn, &err)
+	if n >= bn {
+		// Force failure.
+		if c.extendBuf(n+1) != nil {
+			log.Fatal("expected extension to fail")
+		}
+		return c.Key(s)
+	}
+	return c.extendBuf(n)
+}
+
+// icuUTF8conv implementes the Collator interface.
+// This implentation first converts the give UTF8 string
+// to UTF16 and then calls the main ICU collation function.
+type icuUTF8conv struct {
+	icuCollator
+}
+
+func newUTF8conv(locale string) (Collator, error) {
+	c := &icuUTF8conv{}
+	return c, c.init(locale)
+}
+
+func (c *icuUTF8conv) Compare(sa, sb Input) int {
+	a := encodeUTF16(sa.UTF8)
+	b := encodeUTF16(sb.UTF8)
+	return int(C.ucol_strcoll(c.col, icuUCharP(a), icuULen(a), icuUCharP(b), icuULen(b)))
+}
+
+func (c *icuUTF8conv) Key(s Input) []byte {
+	a := encodeUTF16(s.UTF8)
+	bp, bn := c.buf()
+	n := C.ucol_getSortKey(c.col, icuUCharP(a), icuULen(a), bp, bn)
+	if b := c.extendBuf(n); b != nil {
+		return b
+	}
+	return c.Key(s)
+}
+
+func encodeUTF16(b []byte) []uint16 {
+	a := []uint16{}
+	for len(b) > 0 {
+		r, sz := utf8.DecodeRune(b)
+		b = b[sz:]
+		r1, r2 := utf16.EncodeRune(r)
+		if r1 != 0xFFFD {
+			a = append(a, uint16(r1), uint16(r2))
+		} else {
+			a = append(a, uint16(r))
+		}
+	}
+	return a
+}