diff --git a/go/sql/encoding.go b/go/sql/encoding.go index 767bd9da4..79b638dc4 100644 --- a/go/sql/encoding.go +++ b/go/sql/encoding.go @@ -8,7 +8,10 @@ package sql import ( "golang.org/x/text/encoding" "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" ) type charsetEncoding map[string]encoding.Encoding @@ -20,4 +23,8 @@ func init() { // Begin mappings charsetEncodingMap["latin1"] = charmap.Windows1252 charsetEncodingMap["gbk"] = simplifiedchinese.GBK + charsetEncodingMap["big5"] = traditionalchinese.Big5 + charsetEncodingMap["sjis"] = japanese.ShiftJIS + charsetEncodingMap["ujis"] = japanese.EUCJP + charsetEncodingMap["euckr"] = korean.EUCKR } diff --git a/go/sql/encoding_test.go b/go/sql/encoding_test.go new file mode 100644 index 000000000..259894bc0 --- /dev/null +++ b/go/sql/encoding_test.go @@ -0,0 +1,37 @@ +/* + Copyright 2016 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package sql + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCharsetEncodingMap(t *testing.T) { + testCases := []struct { + name string + charset string + encoded []byte + decoded string + }{ + {name: "Big5", charset: "big5", encoded: []byte{0xa4, 0xa4, 0xa4, 0xe5}, decoded: "中文"}, + {name: "Shift-JIS", charset: "sjis", encoded: []byte{0x93, 0xfa, 0x96, 0x7b}, decoded: "日本"}, + {name: "EUC-JP", charset: "ujis", encoded: []byte{0xc6, 0xfc, 0xcb, 0xdc}, decoded: "日本"}, + {name: "EUC-KR", charset: "euckr", encoded: []byte{0xc7, 0xd1, 0xb1, 0xb9}, decoded: "한국"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + charset, ok := charsetEncodingMap[testCase.charset] + require.True(t, ok) + + decoded, err := charset.NewDecoder().Bytes(testCase.encoded) + require.NoError(t, err) + require.Equal(t, testCase.decoded, string(decoded)) + }) + } +}