initial support for UTF16 input files (#83)

Author: shueybubbles

cmd/sqlcmd/main_test.go

@@ -179,6 +179,55 @@ func TestUnicodeOutput(t *testing.T) {
 	}
 }
 
+func TestUnicodeInput(t *testing.T) {
+	testfiles := []string{
+		`testdata/selectutf8.txt`,
+		`testdata/selectutf8_bom.txt`,
+		`testdata/selectunicode_BE.txt`,
+		`testdata/selectunicode_LE.txt`,
+	}
+
+	for _, test := range testfiles {
+		for _, unicodeOutput := range []bool{true, false} {
+			var outfile string
+			if unicodeOutput {
+				outfile = `testdata/unicodeout_linux.txt`
+				if runtime.GOOS == "windows" {
+					outfile = `testdata/unicodeout.txt`
+				}
+			} else {
+				outfile = `testdata/utf8out_linux.txt`
+				if runtime.GOOS == "windows" {
+					outfile = `testdata/utf8out.txt`
+				}
+			}
+			o, err := os.CreateTemp("", "sqlcmdmain")
+			assert.NoError(t, err, "os.CreateTemp")
+			defer os.Remove(o.Name())
+			defer o.Close()
+			args = newArguments()
+			args.InputFile = []string{test}
+			args.OutputFile = o.Name()
+			args.UnicodeOutputFile = unicodeOutput
+			if canTestAzureAuth() {
+				args.UseAad = true
+			}
+			vars := sqlcmd.InitializeVariables(!args.DisableCmdAndWarn)
+			setVars(vars, &args)
+			exitCode, err := run(vars, &args)
+			assert.NoError(t, err, "run")
+			assert.Equal(t, 0, exitCode, "exitCode")
+			bytes, err := os.ReadFile(o.Name())
+			if assert.NoError(t, err, "os.ReadFile") {
+				expectedBytes, err := os.ReadFile(outfile)
+				if assert.NoErrorf(t, err, "Unable to open %s", outfile) {
+					assert.Equalf(t, expectedBytes, bytes, "input file: <%s> output bytes should match <%s>", test, outfile)
+				}
+			}
+		}
+	}
+}
+
 func TestQueryAndExit(t *testing.T) {
 	o, err := os.CreateTemp("", "sqlcmdmain")
 	assert.NoError(t, err, "os.CreateTemp")

cmd/sqlcmd/testdata/selectunicode_BE.txt

@@ -0,0 +1,2 @@
+select N'挨挨唉哀皑癌蔼矮' as SimplifiedChinese
+

cmd/sqlcmd/testdata/selectunicode_LE.txt

@@ -0,0 +1,5 @@
+SimplifiedChinese
+-----------------
+挨挨唉哀皑癌蔼矮         
+
+(1 row affected)

cmd/sqlcmd/testdata/selectutf8_bom.txt

@@ -0,0 +1,5 @@
+SimplifiedChinese
+-----------------
+挨挨唉哀皑癌蔼矮         
+
+(1 row affected)

cmd/sqlcmd/testdata/utf8out.txt

@@ -21,6 +21,8 @@ import (
 	mssql "github.com/denisenkom/go-mssqldb"
 	"github.com/denisenkom/go-mssqldb/msdsn"
 	"github.com/golang-sql/sqlexp"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 )
 
 var (
@@ -32,6 +34,8 @@ var (
 	ErrCtrlC = errors.New(WarningPrefix + "The last operation was terminated because the user pressed CTRL+C")
 )
 
+const maxLineBuffer = 2 * 1024 * 1024 // 2Mb
+
 // Console defines methods used for console input and output
 type Console interface {
 	// Readline returns the next line of input.
@@ -284,7 +288,11 @@ func (s *Sqlcmd) IncludeFile(path string, processAll bool) error {
 	}
 	defer f.Close()
 	b := s.batch.batchline
-	scanner := bufio.NewScanner(f)
+	utf16bom := unicode.BOMOverride(unicode.UTF8.NewDecoder())
+	unicodeReader := transform.NewReader(f, utf16bom)
+	scanner := bufio.NewScanner(unicodeReader)
+	buf := make([]byte, maxLineBuffer)
+	scanner.Buffer(buf, maxLineBuffer)
 	curLine := s.batch.read
 	s.batch.read = func() (string, error) {
 		if !scanner.Scan() {