implement -u switch (#79)

* implement -u switch

* fix test for linux

Author: shueybubbles

.gitignore

@@ -1,6 +1,6 @@
 # Binaries for programs and plugins
 output
-.exe
+*.exe
 *.exe~
 *.dll
 *.so

README.md

@@ -23,6 +23,7 @@ We will be implementing command line switches and behaviors over time. Several s
   - If `-N` is provided but `-C` is not, sqlcmd will require validation of the server certificate. Note that a `false` value for encryption could still lead to encryption of the login packet.
   - If both `-N` and `-C` are provided, sqlcmd will use their values for encryption negotiation.
   - More information about client/server encryption negotiation can be found at <https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/60f56408-0188-4cd5-8b90-25c6f2423868>
+- `-u` The generated Unicode output file will have the UTF16 Little-Endian Byte-order mark (BOM) written to it.
 - Some behaviors that were kept to maintain compatibility with `OSQL` may be changed, such as alignment of column headers for some data types.
 - All commands must fit on one line, even `EXIT`. Interactive mode will not check for open parentheses or quotes for commands and prompt for successive lines. The ODBC sqlcmd allows the query run by `EXIT(query)` to span multiple lines.
 

cmd/sqlcmd/main.go

@@ -49,8 +49,10 @@ type SQLCmdArguments struct {
 	ErrorLevel                  int               `short:"m" help:"Controls which error messages are sent to stdout. Messages that have severity level greater than or equal to this level are sent."`
 	Format                      string            `short:"F" help:"Specifies the formatting for results." default:"horiz" enum:"horiz,horizontal,vert,vertical"`
 	ErrorsToStderr              int               `short:"r" help:"Redirects the error message output to the screen (stderr). A value of 0 means messages with severity >= 11 will b redirected. A value of 1 means all error message output including PRINT is redirected." enum:"-1,0,1" default:"-1"`
-	Help                        bool              `short:"?" help:"Show syntax summary."`
 	Headers                     int               `short:"h" help:"Specifies the number of rows to print between the column headings. Use -h-1 to specify that headers not be printed."`
+	UnicodeOutputFile           bool              `short:"u" help:"Specifies that all output files are encoded with little-endian Unicode"`
+	// Keep Help at the end of the list
+	Help bool `short:"?" help:"Show syntax summary."`
 }
 
 // Validate accounts for settings not described by Kong attributes
@@ -212,6 +214,7 @@ func run(vars *sqlcmd.Variables, args *SQLCmdArguments) (int, error) {
 	}
 
 	s := sqlcmd.New(line, wd, vars)
+	s.UnicodeOutputFile = args.UnicodeOutputFile
 	setConnect(&s.Connect, args, vars)
 	if args.BatchTerminator != "GO" {
 		err = s.Cmd.SetBatchTerminator(args.BatchTerminator)

cmd/sqlcmd/main_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -80,6 +81,9 @@ func TestValidCommandLineToArgsConversion(t *testing.T) {
 		{[]string{"-h", "2", "-?"}, func(args SQLCmdArguments) bool {
 			return args.Help && args.Headers == 2
 		}},
+		{[]string{"-u"}, func(args SQLCmdArguments) bool {
+			return args.UnicodeOutputFile
+		}},
 	}
 
 	for _, test := range commands {
@@ -144,6 +148,37 @@ func TestRunInputFiles(t *testing.T) {
 	}
 }
 
+func TestUnicodeOutput(t *testing.T) {
+	o, err := os.CreateTemp("", "sqlcmdmain")
+	assert.NoError(t, err, "os.CreateTemp")
+	defer os.Remove(o.Name())
+	defer o.Close()
+	args = newArguments()
+	args.InputFile = []string{"testdata/selectutf8.txt"}
+	args.OutputFile = o.Name()
+	args.UnicodeOutputFile = true
+	if canTestAzureAuth() {
+		args.UseAad = true
+	}
+	vars := sqlcmd.InitializeVariables(!args.DisableCmdAndWarn)
+	setVars(vars, &args)
+
+	exitCode, err := run(vars, &args)
+	assert.NoError(t, err, "run")
+	assert.Equal(t, 0, exitCode, "exitCode")
+	bytes, err := os.ReadFile(o.Name())
+	if assert.NoError(t, err, "os.ReadFile") {
+		outfile := `testdata/unicodeout_linux.txt`
+		if runtime.GOOS == "windows" {
+			outfile = `testdata/unicodeout.txt`
+		}
+		expectedBytes, err := os.ReadFile(outfile)
+		if assert.NoErrorf(t, err, "Unable to open %s", outfile) {
+			assert.Equalf(t, expectedBytes, bytes, "unicode output bytes should match %s", outfile)
+		}
+	}
+}
+
 func TestQueryAndExit(t *testing.T) {
 	o, err := os.CreateTemp("", "sqlcmdmain")
 	assert.NoError(t, err, "os.CreateTemp")

cmd/sqlcmd/testdata/selectutf8.txt

@@ -0,0 +1,2 @@
+select N'挨挨唉哀皑癌蔼矮' as SimplifiedChinese
+

cmd/sqlcmd/testdata/unicodeout.txt

@@ -9,6 +9,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/peterh/liner v1.2.2
 	github.com/stretchr/testify v1.7.1
+	golang.org/x/text v0.3.6
 )
 
 replace github.com/denisenkom/go-mssqldb => github.com/shueybubbles/go-mssqldb v0.10.1-0.20220317022252-fafb9d92e469

cmd/sqlcmd/testdata/unicodeout_linux.txt

@@ -4,10 +4,9 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0 h1:OYa9vmRX2XC5GXRAzegg
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0 h1:v9p9TfTbf7AwNb5NYQt7hI41IfPoLFiFkLtb+bmGjT0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
-github.com/alecthomas/kong v0.2.18-0.20210621093454-54558f65e86f h1:VgRM6/wqZIB1D9W3XMllm/wplTmPgI5yvCHUXEsmKps=
-github.com/alecthomas/kong v0.2.18-0.20210621093454-54558f65e86f/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ=
 github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
 github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
 github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -32,9 +31,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/shueybubbles/go-mssqldb v0.10.1-0.20220317022252-fafb9d92e469 h1:BuUMqsxB86i1QEBf0q+dkQYfNLVpD1nH1fRJPKvXWSg=
 github.com/shueybubbles/go-mssqldb v0.10.1-0.20220317022252-fafb9d92e469/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU=
-github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

go.mod

@@ -11,6 +11,8 @@ import (
 	"strings"
 
 	"github.com/alecthomas/kong"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 )
 
 // Command defines a sqlcmd action which can be intermixed with the SQL batch
@@ -200,7 +202,15 @@ func outCommand(s *Sqlcmd, args []string, line uint) error {
 		if err != nil {
 			return InvalidFileError(err, args[0])
 		}
-		s.SetOutput(o)
+		if s.UnicodeOutputFile {
+			// ODBC sqlcmd doesn't write a BOM but we will.
+			// Maybe the endian-ness should be configurable.
+			win16le := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM)
+			encoder := transform.NewWriter(o, win16le.NewEncoder())
+			s.SetOutput(encoder)
+		} else {
+			s.SetOutput(o)
+		}
 	}
 	return nil
 }