From 61c38a5376f789af7ce70d736bb6be9672a1a3d3 Mon Sep 17 00:00:00 2001 From: sabbott Date: Tue, 7 Jan 2020 17:52:41 -0500 Subject: [PATCH 01/13] fixed a few build errors --- main.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 main.go diff --git a/main.go b/main.go new file mode 100644 index 0000000..984efd1 --- /dev/null +++ b/main.go @@ -0,0 +1,129 @@ +package main + +/* +#include +*/ +import "C" +import ( + "C" + "unsafe" + "database/sql" + "database/sql/driver" + sf "github.com/snowflakedb/gosnowflake" +) + +// TODO free up all c objs esp CString +// TODO figure out if I need to free up the C.char parms +// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) + +// do i need to xport these types? +type connection struct { + db driver.Conn + err *C.char +} + +type statementResult struct { + rowsAffected int64 + err *C.char +} + +// export query +type Query struct { + rows *snowflakeRows + err *C.char +} + +type row struct { + err *C.char + values []*C.char +} + +// export Connect +func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, + user *C.char, password *C.char, role *C.char, port *C.char) *connection { + // other optional parms: Application, Host, and alt auth schemes + cfg := &sf.Config{ + Account: C.GoString(account), + Warehouse: C.GoString(warehouse), + Database: C.GoString(database), + Schema: C.GoString(schema), + User: C.GoString(user), + Password: C.GoString(password), + Role: C.GoString(role), + Region: "us-east-1", + Port: C.GoString(portStr), + } + dsn, err := sf.DSN(cfg) + if err != nil { + return connection{err: C.CString(err.Error())} + } + db, err := sql.Open("snowflake", dsn) + if err != nil { + return connection{err: C.CString(err.Error())} + } + return &connection{db: db} +} + +// export Close +func Close(conn *connection) { + if conn.db != nil { + conn.db.Close() + } + if conn.err != nil { + C.free(unsafe.Pointer(conn.err)) + } +} + +// export Exec +func Exec(conn *connection, statement *C.char) *statementResult { + var res Result + var err error + var result statementResult + + res, err = conn.db.Exec(C.GoString(statement)) + if res != nil { + result.rowsAffected = res.RowsAffected() + } + if err != nil { + result.err = C.CString(err.Error()) + } + return &result +} + +// export Fetch +func Fetch(conn *connection, statement *C.char) *Query { + + rows, err = conn.db.Query(C.GoString(statement)) + result := Query{rows: rows} + + if err != nil { + result.err = C.CString(err.Error()) + } + return &result +} + +//export Next +func Next(queryStruct *Query) *row { + data, err := queryStruct.rows.ChunkDownloader.Next() + //dataLength := len(data) + + // TODO fixme so we set the array length + result := row{} + //result.values = [dataLength]*C.char + //result.values = [4]*C.char + + if err != nil { + result.err = C.CString(err.Error()) + // includes io.EOF + if err == io.EOF { + rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. + } + } + + for i = 0; i < len(data); i++ { + // TODO figure out if I need to handle db NULL differently + result.values[i] = C.CString(data[i]) + } + return &result +} + From 4b96ed7a0db5cafe448154086b87722773bf508d Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 8 Jan 2020 13:00:47 -0500 Subject: [PATCH 02/13] Store db connection and last error in global vars. Return simpler results. --- main.go | 129 ------------------ ruby_snowflake_connector/.gitignore | 1 + .../src/snowflake/main.go | 102 ++++++++++++++ 3 files changed, 103 insertions(+), 129 deletions(-) delete mode 100644 main.go create mode 100644 ruby_snowflake_connector/.gitignore create mode 100644 ruby_snowflake_connector/src/snowflake/main.go diff --git a/main.go b/main.go deleted file mode 100644 index 984efd1..0000000 --- a/main.go +++ /dev/null @@ -1,129 +0,0 @@ -package main - -/* -#include -*/ -import "C" -import ( - "C" - "unsafe" - "database/sql" - "database/sql/driver" - sf "github.com/snowflakedb/gosnowflake" -) - -// TODO free up all c objs esp CString -// TODO figure out if I need to free up the C.char parms -// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) - -// do i need to xport these types? -type connection struct { - db driver.Conn - err *C.char -} - -type statementResult struct { - rowsAffected int64 - err *C.char -} - -// export query -type Query struct { - rows *snowflakeRows - err *C.char -} - -type row struct { - err *C.char - values []*C.char -} - -// export Connect -func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, - user *C.char, password *C.char, role *C.char, port *C.char) *connection { - // other optional parms: Application, Host, and alt auth schemes - cfg := &sf.Config{ - Account: C.GoString(account), - Warehouse: C.GoString(warehouse), - Database: C.GoString(database), - Schema: C.GoString(schema), - User: C.GoString(user), - Password: C.GoString(password), - Role: C.GoString(role), - Region: "us-east-1", - Port: C.GoString(portStr), - } - dsn, err := sf.DSN(cfg) - if err != nil { - return connection{err: C.CString(err.Error())} - } - db, err := sql.Open("snowflake", dsn) - if err != nil { - return connection{err: C.CString(err.Error())} - } - return &connection{db: db} -} - -// export Close -func Close(conn *connection) { - if conn.db != nil { - conn.db.Close() - } - if conn.err != nil { - C.free(unsafe.Pointer(conn.err)) - } -} - -// export Exec -func Exec(conn *connection, statement *C.char) *statementResult { - var res Result - var err error - var result statementResult - - res, err = conn.db.Exec(C.GoString(statement)) - if res != nil { - result.rowsAffected = res.RowsAffected() - } - if err != nil { - result.err = C.CString(err.Error()) - } - return &result -} - -// export Fetch -func Fetch(conn *connection, statement *C.char) *Query { - - rows, err = conn.db.Query(C.GoString(statement)) - result := Query{rows: rows} - - if err != nil { - result.err = C.CString(err.Error()) - } - return &result -} - -//export Next -func Next(queryStruct *Query) *row { - data, err := queryStruct.rows.ChunkDownloader.Next() - //dataLength := len(data) - - // TODO fixme so we set the array length - result := row{} - //result.values = [dataLength]*C.char - //result.values = [4]*C.char - - if err != nil { - result.err = C.CString(err.Error()) - // includes io.EOF - if err == io.EOF { - rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. - } - } - - for i = 0; i < len(data); i++ { - // TODO figure out if I need to handle db NULL differently - result.values[i] = C.CString(data[i]) - } - return &result -} - diff --git a/ruby_snowflake_connector/.gitignore b/ruby_snowflake_connector/.gitignore new file mode 100644 index 0000000..d8fe4fa --- /dev/null +++ b/ruby_snowflake_connector/.gitignore @@ -0,0 +1 @@ +/.project diff --git a/ruby_snowflake_connector/src/snowflake/main.go b/ruby_snowflake_connector/src/snowflake/main.go new file mode 100644 index 0000000..ce1d216 --- /dev/null +++ b/ruby_snowflake_connector/src/snowflake/main.go @@ -0,0 +1,102 @@ +package main + +/* +#include +*/ +import "C" +import ( + "database/sql" + "database/sql/driver" + sf "github.com/snowflakedb/gosnowflake" +) + +// TODO free up all c objs esp CString +// TODO figure out if I need to free up the C.char parms +// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) + +// Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them +// back and forth to ruby +var last_error error +var db driver.Conn + +// export LastError +func LastError() *C.char { + if last_error == nil { + return nil + } else { + return C.CString(last_error.Error()) + } +} + +// @returns nil if no error or the error string +// export Connect +func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, + user *C.char, password *C.char, role *C.char, port *C.char) *C.char { + // other optional parms: Application, Host, and alt auth schemes + cfg := &sf.Config{ + Account: C.GoString(account), + Warehouse: C.GoString(warehouse), + Database: C.GoString(database), + Schema: C.GoString(schema), + User: C.GoString(user), + Password: C.GoString(password), + Role: C.GoString(role), + Region: "us-east-1", + Port: C.GoString(portStr), + } + dsn, last_error := sf.DSN(cfg) + if last_error != nil { + return LastError() + } + db, last_error = sql.Open("snowflake", dsn) + if last_error != nil { + return LastError() + } + return nil +} + +// export Close +func Close() { + if db != nil { + db.Close() + } +} + +// export Exec +func Exec(statement *C.char) int64 { + var res Result + + res, last_error = db.Exec(C.GoString(statement)) + if res != nil { + return res.RowsAffected() + } + return nil +} + +// export Fetch +func Fetch(statement *C.char) *snowflakeRows { + + rows, last_error := db.Query(C.GoString(statement)) + return rows +} + +//export Next +func Next(rows *snowflakeRows) []*C.char { + data, last_error := rows.ChunkDownloader.Next() + + // includes io.EOF + if last_error == io.EOF { + rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. + } + + if data != nil { + result := [len(data)]*C.char + + for i = 0; i < len(data); i++ { + // TODO figure out if I need to handle db NULL differently + result[i] = C.CString(data[i]) + } + return &result[0] + } + return nil +} From c5ed025a042bb8d274fd9c87992915829646c9e6 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 8 Jan 2020 13:36:20 -0500 Subject: [PATCH 03/13] foo --- .gitignore | 1 + main.go | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 .gitignore create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d8fe4fa --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.project diff --git a/main.go b/main.go new file mode 100644 index 0000000..ce1d216 --- /dev/null +++ b/main.go @@ -0,0 +1,102 @@ +package main + +/* +#include +*/ +import "C" +import ( + "database/sql" + "database/sql/driver" + sf "github.com/snowflakedb/gosnowflake" +) + +// TODO free up all c objs esp CString +// TODO figure out if I need to free up the C.char parms +// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) + +// Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them +// back and forth to ruby +var last_error error +var db driver.Conn + +// export LastError +func LastError() *C.char { + if last_error == nil { + return nil + } else { + return C.CString(last_error.Error()) + } +} + +// @returns nil if no error or the error string +// export Connect +func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, + user *C.char, password *C.char, role *C.char, port *C.char) *C.char { + // other optional parms: Application, Host, and alt auth schemes + cfg := &sf.Config{ + Account: C.GoString(account), + Warehouse: C.GoString(warehouse), + Database: C.GoString(database), + Schema: C.GoString(schema), + User: C.GoString(user), + Password: C.GoString(password), + Role: C.GoString(role), + Region: "us-east-1", + Port: C.GoString(portStr), + } + dsn, last_error := sf.DSN(cfg) + if last_error != nil { + return LastError() + } + db, last_error = sql.Open("snowflake", dsn) + if last_error != nil { + return LastError() + } + return nil +} + +// export Close +func Close() { + if db != nil { + db.Close() + } +} + +// export Exec +func Exec(statement *C.char) int64 { + var res Result + + res, last_error = db.Exec(C.GoString(statement)) + if res != nil { + return res.RowsAffected() + } + return nil +} + +// export Fetch +func Fetch(statement *C.char) *snowflakeRows { + + rows, last_error := db.Query(C.GoString(statement)) + return rows +} + +//export Next +func Next(rows *snowflakeRows) []*C.char { + data, last_error := rows.ChunkDownloader.Next() + + // includes io.EOF + if last_error == io.EOF { + rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. + } + + if data != nil { + result := [len(data)]*C.char + + for i = 0; i < len(data); i++ { + // TODO figure out if I need to handle db NULL differently + result[i] = C.CString(data[i]) + } + return &result[0] + } + return nil +} From 3a660eac9c2b43680f47bdfaa62a23812298e597 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 8 Jan 2020 13:36:20 -0500 Subject: [PATCH 04/13] Go ruby snowflake client --- .gitignore | 4 + Gemfile | 4 + LICENSE.txt | 21 +++ README.md | 0 Rakefile | 1 + ext/Makefile | 6 + ext/extconf.rb | 1 + ext/ruby_snowflake.go | 149 ++++++++++++++++++ lib/go_snowflake_client.rb | 88 +++++++++++ lib/ruby_snowflake_client/version.rb | 3 + main.go | 102 ------------ ruby_snowflake_client.gemspec | 26 +++ ruby_snowflake_connector/.gitignore | 1 - .../src/snowflake/main.go | 102 ------------ 14 files changed, 303 insertions(+), 205 deletions(-) create mode 100644 Gemfile create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100644 ext/Makefile create mode 100644 ext/extconf.rb create mode 100644 ext/ruby_snowflake.go create mode 100644 lib/go_snowflake_client.rb create mode 100644 lib/ruby_snowflake_client/version.rb delete mode 100644 main.go create mode 100644 ruby_snowflake_client.gemspec delete mode 100644 ruby_snowflake_connector/.gitignore delete mode 100644 ruby_snowflake_connector/src/snowflake/main.go diff --git a/.gitignore b/.gitignore index d8fe4fa..07874ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ /.project +ruby_snowflake_client.h +ruby_snowflake_client.so +/.rakeTasks +.idea/* diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..396c3dc --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in scatter.gemspec +gemspec diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..e3c3ff6 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Dotan Nahum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..2995527 --- /dev/null +++ b/Rakefile @@ -0,0 +1 @@ +require "bundler/gem_tasks" diff --git a/ext/Makefile b/ext/Makefile new file mode 100644 index 0000000..1278c19 --- /dev/null +++ b/ext/Makefile @@ -0,0 +1,6 @@ +build: + go build -buildmode=c-shared -o ruby_snowflake_client.so ruby_snowflake.go +clean: +install: + +.PHONY: build diff --git a/ext/extconf.rb b/ext/extconf.rb new file mode 100644 index 0000000..f7b3cad --- /dev/null +++ b/ext/extconf.rb @@ -0,0 +1 @@ +`make build` diff --git a/ext/ruby_snowflake.go b/ext/ruby_snowflake.go new file mode 100644 index 0000000..68d9038 --- /dev/null +++ b/ext/ruby_snowflake.go @@ -0,0 +1,149 @@ +package main + +/* +#include +*/ +import "C" +import ( + "database/sql" + "errors" + sf "github.com/snowflakedb/gosnowflake" + "unsafe" + "io" + gopointer "github.com/mattn/go-pointer" +// "fmt" +) + +// TODO free up all c objs esp CString +// TODO Close the query (I think it's a noop tho) + +// Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them +// back and forth to ruby +var last_error error +var db *sql.DB // TODO follow gopointer pattern to return this to ruby + +//export LastError +func LastError() *C.char { + if last_error == nil { + return nil + } else { + return C.CString(last_error.Error()) + } +} + +// @returns nil if no error or the error string +// ugh, ruby and go were disagreeing about the length of `int` so I had to be particular here and in the ffi +//export Connect +func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, + user *C.char, password *C.char, role *C.char, port int64) *C.char { + // other optional parms: Application, Host, and alt auth schemes + cfg := &sf.Config{ + Account: C.GoString(account), + Warehouse: C.GoString(warehouse), + Database: C.GoString(database), + Schema: C.GoString(schema), + User: C.GoString(user), + Password: C.GoString(password), + Role: C.GoString(role), + Region: "us-east-1", + Port: int(port), + } + + dsn, last_error := sf.DSN(cfg) + if last_error != nil { + return LastError() + } + + db, last_error = sql.Open("snowflake", dsn) + if last_error != nil { + return LastError() + } + + return nil +} + +//export Close +func Close() { + if db != nil { + db.Close() + } +} + +// @return number of rows affected or -1 for error +//export Exec +func Exec(statement *C.char) int64 { + var res sql.Result + res, last_error = db.Exec(C.GoString(statement)) + if res != nil { + rows, _ := res.RowsAffected() + return rows + } + return -1 +} + +//export Fetch +func Fetch(statement *C.char) unsafe.Pointer { + var rows *sql.Rows + rows, last_error = db.Query(C.GoString(statement)) + if rows != nil { + result := gopointer.Save(rows) + return result + } else { + return nil + } +} + +// NOTE: gc's the rows_pointer object on EOF and returns nil. LastError is set to EOF +// may need to be **C.char? +//export NextRow +func NextRow(rows_pointer unsafe.Pointer) **C.char { + decode := gopointer.Restore(rows_pointer) + var rows *sql.Rows + + if decode != nil { + rows = decode.(*sql.Rows) + } else { + last_error = errors.New("rows_pointer invalid: Restore returned nil") + return nil + } + + if rows.Next() { + columns, _ := rows.Columns() + rowLength := len(columns) + + rawResult := make([][]byte, rowLength) + rawData := make([]interface{}, rowLength) + for i, _ := range rawResult { // found in stackoverflow, fwiw + rawData[i] = &rawResult[i] // Put pointers to each string in the interface slice + } + + // https://stackoverflow.com/questions/58866962/how-to-pass-an-array-of-strings-and-get-an-array-of-strings-in-ruby-using-go-sha + pointerSize := unsafe.Sizeof(rows_pointer) + // Allocate an array for the string pointers. + var out **C.char + out = (**C.char)(C.malloc(C.ulong(rowLength) * C.ulong(pointerSize))) + + last_error = rows.Scan(rawData...) + if last_error != nil { + return nil + } + pointer := out + for _, raw := range rawResult { + // Find where to store the address of the next string. + // Copy each output string to a C string, and add it to the array. + // C.CString uses malloc to allocate memory. + if raw == nil { + *pointer = nil + } else { + *pointer = C.CString(string(raw)) + } + pointer = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(pointer)) + pointerSize)) + } + return out + } else if rows.Err() == io.EOF { + gopointer.Unref(rows_pointer) // free up for gc + } + return nil +} + +func main(){} diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb new file mode 100644 index 0000000..a517c91 --- /dev/null +++ b/lib/go_snowflake_client.rb @@ -0,0 +1,88 @@ +$: << '.' +require 'ruby_snowflake_client/version' +require 'ffi' + +# Note: this library is not thread safe as it caches the db and last error +# The call pattern expectation is to call last_error after any call which may have gotten an error. If last_error is +# `nil`, there was no error. The exception is `connect` which currently just returns the error or `nil`. +module GoSnowflakeClient + extend self + + # @return String last error or nil. May be end of file which is not really an error + def last_error() + error, cptr = GoSnowflakeClientBinding.last_error + LibC.free(cptr) if error + error + end + + # @param account[String] should include everything in the db url ahead of region.snowflakecomputing.com + # @param port[Integer] + # @return error[String] or nil + def connect(account, warehouse, database, schema, user, password, role, port = 443) + error, cptr = GoSnowflakeClientBinding.connect(account, warehouse, database, schema, user, password, role, port) + LibC.free(cptr) if error + error + end + + def close + GoSnowflakeClientBinding.close() + end + + # @param statement[String] an executable query which should return number of rows affected + # @return rowcount[Number] number of rows or nil if there was an error + def exec(statement) + count = GoSnowflakeClientBinding.exec(statement) # returns -1 for error + count >= 0 ? count : nil + end + + # @return query_object[Pointer] a pointer to use for subsequent calls not inspectable nor viewable by Ruby; however, + # if it's `nil`, check `last_error` + def fetch(query) + GoSnowflakeClientBinding.fetch(query) + end + + # @param query_object[Pointer] the pointer which `fetch` returned. Go will gc this object when the query is done; so, + # don't expect to reference it after the call which returned `nil` + # @return [List] the column values in order + def get_next_row(query_object, field_count) + raw_row = GoSnowflakeClientBinding.next_row(query_object) + return nil if raw_row.nil? || raw_row == FFI::Pointer::NULL + + raw_row.get_array_of_pointer(0, field_count).map do |cstr| + if cstr == FFI::Pointer::NULL + nil + else + str = cstr.read_string + LibC.free(cstr) + str + end + end + ensure + LibC.free(raw_row) if raw_row + end + + # TODO write query method which takes block and iterates with an ensure to tell go to release query_object and that + # takes a list of converters for casting strings to intended types + + module LibC + extend FFI::Library + ffi_lib(FFI::Library::LIBC) + + attach_function(:free, [:pointer], :void) + end + + module GoSnowflakeClientBinding + extend FFI::Library + + POINTER_SIZE = FFI.type_size(:pointer) + + ffi_lib(File.expand_path('../ext/ruby_snowflake_client.so', File.dirname(__FILE__))) + attach_function(:last_error, 'LastError', [], :strptr) + # ugh, `port` in gosnowflake is just :int; however, ruby - ffi -> go is passing 32bit int if I just decl :int. + attach_function(:connect, 'Connect', [:string, :string, :string, :string, :string, :string, :string, :int64], :strptr) + attach_function(:close, 'Close', [], :void) + attach_function(:exec, 'Exec', [:string], :int64) + attach_function(:fetch, 'Fetch', [:string], :pointer) + attach_function(:next_row, 'NextRow', [:pointer], :pointer) + end +end diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb new file mode 100644 index 0000000..14bfe13 --- /dev/null +++ b/lib/ruby_snowflake_client/version.rb @@ -0,0 +1,3 @@ +module GoSnowflakeClient + VERSION = "0.2.1" +end diff --git a/main.go b/main.go deleted file mode 100644 index ce1d216..0000000 --- a/main.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -/* -#include -*/ -import "C" -import ( - "database/sql" - "database/sql/driver" - sf "github.com/snowflakedb/gosnowflake" -) - -// TODO free up all c objs esp CString -// TODO figure out if I need to free up the C.char parms -// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) - -// Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them -// back and forth to ruby -var last_error error -var db driver.Conn - -// export LastError -func LastError() *C.char { - if last_error == nil { - return nil - } else { - return C.CString(last_error.Error()) - } -} - -// @returns nil if no error or the error string -// export Connect -func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, - user *C.char, password *C.char, role *C.char, port *C.char) *C.char { - // other optional parms: Application, Host, and alt auth schemes - cfg := &sf.Config{ - Account: C.GoString(account), - Warehouse: C.GoString(warehouse), - Database: C.GoString(database), - Schema: C.GoString(schema), - User: C.GoString(user), - Password: C.GoString(password), - Role: C.GoString(role), - Region: "us-east-1", - Port: C.GoString(portStr), - } - dsn, last_error := sf.DSN(cfg) - if last_error != nil { - return LastError() - } - db, last_error = sql.Open("snowflake", dsn) - if last_error != nil { - return LastError() - } - return nil -} - -// export Close -func Close() { - if db != nil { - db.Close() - } -} - -// export Exec -func Exec(statement *C.char) int64 { - var res Result - - res, last_error = db.Exec(C.GoString(statement)) - if res != nil { - return res.RowsAffected() - } - return nil -} - -// export Fetch -func Fetch(statement *C.char) *snowflakeRows { - - rows, last_error := db.Query(C.GoString(statement)) - return rows -} - -//export Next -func Next(rows *snowflakeRows) []*C.char { - data, last_error := rows.ChunkDownloader.Next() - - // includes io.EOF - if last_error == io.EOF { - rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. - } - - if data != nil { - result := [len(data)]*C.char - - for i = 0; i < len(data); i++ { - // TODO figure out if I need to handle db NULL differently - result[i] = C.CString(data[i]) - } - return &result[0] - } - return nil -} diff --git a/ruby_snowflake_client.gemspec b/ruby_snowflake_client.gemspec new file mode 100644 index 0000000..2f917ae --- /dev/null +++ b/ruby_snowflake_client.gemspec @@ -0,0 +1,26 @@ +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'ruby_snowflake_client/version' + +Gem::Specification.new do |s| + s.name = "ruby_snowflake_client" + s.version = GoSnowflakeClient::VERSION + s.summary = "Snowflake connect for Ruby" + s.author = "CarGurus" + s.email = ['dmitchell@cargurus.com', 'sabbott@cargurus.com'] + s.platform = Gem::Platform::CURRENT + s.description = <<~DESC + Uses gosnowflake to connect to and communicate with Snowflake. + This library is much faster than using ODBC especially for large result sets and avoids ODBC butchering of timezones. + DESC + s.license = 'MIT' # TODO double check + + s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb'] + + # perhaps nothing and figure out how to build and pkg the platform specific .so, or .a, or ... + s.extensions << "ext/extconf.rb" + + s.add_dependency 'ffi' + s.add_development_dependency "bundler" + s.add_development_dependency "rake" +end diff --git a/ruby_snowflake_connector/.gitignore b/ruby_snowflake_connector/.gitignore deleted file mode 100644 index d8fe4fa..0000000 --- a/ruby_snowflake_connector/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/.project diff --git a/ruby_snowflake_connector/src/snowflake/main.go b/ruby_snowflake_connector/src/snowflake/main.go deleted file mode 100644 index ce1d216..0000000 --- a/ruby_snowflake_connector/src/snowflake/main.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -/* -#include -*/ -import "C" -import ( - "database/sql" - "database/sql/driver" - sf "github.com/snowflakedb/gosnowflake" -) - -// TODO free up all c objs esp CString -// TODO figure out if I need to free up the C.char parms -// TODO Close the query (I think it's a noop but that'd be a good place to free up CString) - -// Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them -// back and forth to ruby -var last_error error -var db driver.Conn - -// export LastError -func LastError() *C.char { - if last_error == nil { - return nil - } else { - return C.CString(last_error.Error()) - } -} - -// @returns nil if no error or the error string -// export Connect -func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, - user *C.char, password *C.char, role *C.char, port *C.char) *C.char { - // other optional parms: Application, Host, and alt auth schemes - cfg := &sf.Config{ - Account: C.GoString(account), - Warehouse: C.GoString(warehouse), - Database: C.GoString(database), - Schema: C.GoString(schema), - User: C.GoString(user), - Password: C.GoString(password), - Role: C.GoString(role), - Region: "us-east-1", - Port: C.GoString(portStr), - } - dsn, last_error := sf.DSN(cfg) - if last_error != nil { - return LastError() - } - db, last_error = sql.Open("snowflake", dsn) - if last_error != nil { - return LastError() - } - return nil -} - -// export Close -func Close() { - if db != nil { - db.Close() - } -} - -// export Exec -func Exec(statement *C.char) int64 { - var res Result - - res, last_error = db.Exec(C.GoString(statement)) - if res != nil { - return res.RowsAffected() - } - return nil -} - -// export Fetch -func Fetch(statement *C.char) *snowflakeRows { - - rows, last_error := db.Query(C.GoString(statement)) - return rows -} - -//export Next -func Next(rows *snowflakeRows) []*C.char { - data, last_error := rows.ChunkDownloader.Next() - - // includes io.EOF - if last_error == io.EOF { - rows.ChunkDownloader.Chunks = nil // detach all chunks. No way to go backward without reinitialize it. - } - - if data != nil { - result := [len(data)]*C.char - - for i = 0; i < len(data); i++ { - // TODO figure out if I need to handle db NULL differently - result[i] = C.CString(data[i]) - } - return &result[0] - } - return nil -} From d278bdb11683e0dcf455e9f0effe3befe4ea2bb1 Mon Sep 17 00:00:00 2001 From: sabbott Date: Mon, 13 Jan 2020 16:16:29 -0500 Subject: [PATCH 05/13] first hack at a working gem (for linux, at least) --- .gitignore | 3 +++ Gemfile.lock | 22 ++++++++++++++++++++++ ext/extconf.rb | 1 - ruby_snowflake_client.gemspec | 4 ++-- 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 07874ca..24cb486 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ ruby_snowflake_client.h ruby_snowflake_client.so /.rakeTasks .idea/* + +# ruby gems +*.gem diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..ed8a098 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,22 @@ +PATH + remote: . + specs: + ruby_snowflake_client (0.2.1-x86_64-linux) + ffi + +GEM + remote: https://rubygems.org/ + specs: + ffi (1.11.3) + rake (13.0.1) + +PLATFORMS + ruby + +DEPENDENCIES + bundler + rake + ruby_snowflake_client! + +BUNDLED WITH + 1.17.3 diff --git a/ext/extconf.rb b/ext/extconf.rb index f7b3cad..e69de29 100644 --- a/ext/extconf.rb +++ b/ext/extconf.rb @@ -1 +0,0 @@ -`make build` diff --git a/ruby_snowflake_client.gemspec b/ruby_snowflake_client.gemspec index 2f917ae..c1d707c 100644 --- a/ruby_snowflake_client.gemspec +++ b/ruby_snowflake_client.gemspec @@ -15,10 +15,10 @@ Gem::Specification.new do |s| DESC s.license = 'MIT' # TODO double check - s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb'] + s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb', 'lib/ruby_snowflake_client/version.rb'] # perhaps nothing and figure out how to build and pkg the platform specific .so, or .a, or ... - s.extensions << "ext/extconf.rb" + #s.extensions << "ext/ruby_snowflake_client/extconf.rb" s.add_dependency 'ffi' s.add_development_dependency "bundler" From 237c455b0cdfbfc6150650af1e5fd7b069668842 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Mon, 13 Jan 2020 16:36:29 -0500 Subject: [PATCH 06/13] Default port id --- lib/go_snowflake_client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index a517c91..f6ece09 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -1,4 +1,4 @@ -$: << '.' +$LOAD_PATH << File.dirname(__FILE__) require 'ruby_snowflake_client/version' require 'ffi' @@ -19,7 +19,7 @@ def last_error() # @param port[Integer] # @return error[String] or nil def connect(account, warehouse, database, schema, user, password, role, port = 443) - error, cptr = GoSnowflakeClientBinding.connect(account, warehouse, database, schema, user, password, role, port) + error, cptr = GoSnowflakeClientBinding.connect(account, warehouse, database, schema, user, password, role, port || 443) LibC.free(cptr) if error error end @@ -49,7 +49,7 @@ def get_next_row(query_object, field_count) return nil if raw_row.nil? || raw_row == FFI::Pointer::NULL raw_row.get_array_of_pointer(0, field_count).map do |cstr| - if cstr == FFI::Pointer::NULL + if cstr == FFI::Pointer::NULL || cstr.nil? nil else str = cstr.read_string From 06235372ea1b2d3f1dd63bb66eeb39b8fd2f5ccf Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Tue, 14 Jan 2020 13:56:12 -0500 Subject: [PATCH 07/13] Pass db pointer back to make the lib a little less thread averse. last_error is still a global tho --- .gitignore | 1 + Gemfile.lock | 2 +- ext/ruby_snowflake.go | 47 +++++++++++++++++----------- lib/go_snowflake_client.rb | 32 ++++++++++--------- lib/ruby_snowflake_client/version.rb | 2 +- 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 24cb486..f6b4e5b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ ruby_snowflake_client.so # ruby gems *.gem +/.DS_Store diff --git a/Gemfile.lock b/Gemfile.lock index ed8a098..dc6b551 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ruby_snowflake_client (0.2.1-x86_64-linux) + ruby_snowflake_client (0.2.1-x86_64-darwin-18) ffi GEM diff --git a/ext/ruby_snowflake.go b/ext/ruby_snowflake.go index 68d9038..63d8103 100644 --- a/ext/ruby_snowflake.go +++ b/ext/ruby_snowflake.go @@ -14,13 +14,9 @@ import ( // "fmt" ) -// TODO free up all c objs esp CString -// TODO Close the query (I think it's a noop tho) - // Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them // back and forth to ruby var last_error error -var db *sql.DB // TODO follow gopointer pattern to return this to ruby //export LastError func LastError() *C.char { @@ -31,11 +27,11 @@ func LastError() *C.char { } } -// @returns nil if no error or the error string +// @returns db pointer // ugh, ruby and go were disagreeing about the length of `int` so I had to be particular here and in the ffi //export Connect func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, - user *C.char, password *C.char, role *C.char, port int64) *C.char { + user *C.char, password *C.char, role *C.char, port int64) unsafe.Pointer { // other optional parms: Application, Host, and alt auth schemes cfg := &sf.Config{ Account: C.GoString(account), @@ -51,19 +47,21 @@ func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.cha dsn, last_error := sf.DSN(cfg) if last_error != nil { - return LastError() + return nil } + var db *sql.DB db, last_error = sql.Open("snowflake", dsn) - if last_error != nil { - return LastError() + if db == nil { + return nil + } else { + return gopointer.Save(db) } - - return nil } //export Close -func Close() { +func Close(db_pointer unsafe.Pointer) { + db := decodeDbPointer(db_pointer) if db != nil { db.Close() } @@ -71,7 +69,8 @@ func Close() { // @return number of rows affected or -1 for error //export Exec -func Exec(statement *C.char) int64 { +func Exec(db_pointer unsafe.Pointer, statement *C.char) int64 { + db := decodeDbPointer(db_pointer) var res sql.Result res, last_error = db.Exec(C.GoString(statement)) if res != nil { @@ -82,7 +81,8 @@ func Exec(statement *C.char) int64 { } //export Fetch -func Fetch(statement *C.char) unsafe.Pointer { +func Fetch(db_pointer unsafe.Pointer, statement *C.char) unsafe.Pointer { + db := decodeDbPointer(db_pointer) var rows *sql.Rows rows, last_error = db.Query(C.GoString(statement)) if rows != nil { @@ -94,15 +94,16 @@ func Fetch(statement *C.char) unsafe.Pointer { } // NOTE: gc's the rows_pointer object on EOF and returns nil. LastError is set to EOF -// may need to be **C.char? //export NextRow func NextRow(rows_pointer unsafe.Pointer) **C.char { - decode := gopointer.Restore(rows_pointer) + if rows_pointer == nil { + last_error = errors.New("rows_pointer null: cannot fetch") + return nil + } var rows *sql.Rows + rows = gopointer.Restore(rows_pointer).(*sql.Rows) - if decode != nil { - rows = decode.(*sql.Rows) - } else { + if rows == nil { last_error = errors.New("rows_pointer invalid: Restore returned nil") return nil } @@ -146,4 +147,12 @@ func NextRow(rows_pointer unsafe.Pointer) **C.char { return nil } +func decodeDbPointer(db_pointer unsafe.Pointer) *sql.DB { + if db_pointer == nil { + last_error = errors.New("db_pointer is null. Cannot process command.") + return nil + } + return gopointer.Restore(db_pointer).(*sql.DB) +} + func main(){} diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index f6ece09..6f54846 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -4,7 +4,7 @@ # Note: this library is not thread safe as it caches the db and last error # The call pattern expectation is to call last_error after any call which may have gotten an error. If last_error is -# `nil`, there was no error. The exception is `connect` which currently just returns the error or `nil`. +# `nil`, there was no error. module GoSnowflakeClient extend self @@ -17,28 +17,30 @@ def last_error() # @param account[String] should include everything in the db url ahead of region.snowflakecomputing.com # @param port[Integer] - # @return error[String] or nil + # @return query_object[Pointer] a pointer to use for subsequent calls not inspectable nor viewable by Ruby def connect(account, warehouse, database, schema, user, password, role, port = 443) - error, cptr = GoSnowflakeClientBinding.connect(account, warehouse, database, schema, user, password, role, port || 443) - LibC.free(cptr) if error - error + GoSnowflakeClientBinding.connect(account, warehouse, database, schema, user, password, role, port || 443) end - def close - GoSnowflakeClientBinding.close() + # @param db_pointer[Pointer] the pointer which `connect` returned. + def close(db_pointer) + GoSnowflakeClientBinding.close(db_pointer) end + # @param db_pointer[Pointer] the pointer which `connect` returned. # @param statement[String] an executable query which should return number of rows affected # @return rowcount[Number] number of rows or nil if there was an error - def exec(statement) - count = GoSnowflakeClientBinding.exec(statement) # returns -1 for error + def exec(db_pointer, statement) + count = GoSnowflakeClientBinding.exec(db_pointer, statement) # returns -1 for error count >= 0 ? count : nil end + # @param db_pointer[Pointer] the pointer which `connect` returned. + # @param query[String] a select query to run. # @return query_object[Pointer] a pointer to use for subsequent calls not inspectable nor viewable by Ruby; however, # if it's `nil`, check `last_error` - def fetch(query) - GoSnowflakeClientBinding.fetch(query) + def fetch(db_pointer, query) + GoSnowflakeClientBinding.fetch(db_pointer, query) end # @param query_object[Pointer] the pointer which `fetch` returned. Go will gc this object when the query is done; so, @@ -79,10 +81,10 @@ module GoSnowflakeClientBinding ffi_lib(File.expand_path('../ext/ruby_snowflake_client.so', File.dirname(__FILE__))) attach_function(:last_error, 'LastError', [], :strptr) # ugh, `port` in gosnowflake is just :int; however, ruby - ffi -> go is passing 32bit int if I just decl :int. - attach_function(:connect, 'Connect', [:string, :string, :string, :string, :string, :string, :string, :int64], :strptr) - attach_function(:close, 'Close', [], :void) - attach_function(:exec, 'Exec', [:string], :int64) - attach_function(:fetch, 'Fetch', [:string], :pointer) + attach_function(:connect, 'Connect', [:string, :string, :string, :string, :string, :string, :string, :int64], :pointer) + attach_function(:close, 'Close', [:pointer], :void) + attach_function(:exec, 'Exec', [:pointer, :string], :int64) + attach_function(:fetch, 'Fetch', [:pointer, :string], :pointer) attach_function(:next_row, 'NextRow', [:pointer], :pointer) end end diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb index 14bfe13..83be6df 100644 --- a/lib/ruby_snowflake_client/version.rb +++ b/lib/ruby_snowflake_client/version.rb @@ -1,3 +1,3 @@ module GoSnowflakeClient - VERSION = "0.2.1" + VERSION = "0.2.2" end From 42dab78f89a5d835e22b063a700fec2cde3eef5f Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Tue, 14 Jan 2020 17:54:04 -0500 Subject: [PATCH 08/13] Examples and simplified select pattern --- Gemfile.lock | 2 +- examples/snowflake_sample_data.rb | 57 ++++++ ext/ruby_snowflake.go | 276 ++++++++++++++++----------- lib/go_snowflake_client.rb | 50 +++++ lib/ruby_snowflake_client/version.rb | 2 +- 5 files changed, 271 insertions(+), 116 deletions(-) create mode 100644 examples/snowflake_sample_data.rb diff --git a/Gemfile.lock b/Gemfile.lock index dc6b551..e897a21 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ruby_snowflake_client (0.2.1-x86_64-darwin-18) + ruby_snowflake_client (0.2.3-x86_64-darwin-18) ffi GEM diff --git a/examples/snowflake_sample_data.rb b/examples/snowflake_sample_data.rb new file mode 100644 index 0000000..ba31f9f --- /dev/null +++ b/examples/snowflake_sample_data.rb @@ -0,0 +1,57 @@ +# Assumes you have access to snowflake_sample_data https://docs.snowflake.net/manuals/user-guide/sample-data.html +# Set env vars: SNOWFLAKE_TEST_ACCOUNT, SNOWFLAKE_TEST_USER, SNOWFLAKE_TEST_PASSWORD, SNOWFLAKE_TEST_WAREHOUSE +# optionally set SNOWFLAKE_TEST_SCHEMA, SNOWFLAKE_TEST_ROLE +$LOAD_PATH << File.expand_path('../..', __FILE__) +require 'lib/go_snowflake_client' +require 'logger' + +class SnowflakeSampleData + + def initialize + @logger = Logger.new(STDERR) + + @db_pointer = GoSnowflakeClient.connect( + ENV['SNOWFLAKE_TEST_ACCOUNT'], + ENV['SNOWFLAKE_TEST_WAREHOUSE'], + "SNOWFLAKE_SAMPLE_DATA", + ENV['SNOWFLAKE_TEST_SCHEMA'] || 'TPCDS_SF10TCL', + ENV['SNOWFLAKE_TEST_USER'], + ENV['SNOWFLAKE_TEST_PASSWORD'], + ENV['SNOWFLAKE_TEST_ROLE'] || 'PUBLIC') + + log_error unless @db_pointer + end + + def close_db + GoSnowflakeClient.close(@db_pointer) if @db_pointer + end + + def get_customer_names(where = "c_last_name = 'Flowers'") + raise('db not connected') unless @db_pointer + + query = "select c_first_name, c_last_name from \"CUSTOMER\"" + query += " where #{where}" if where + + GoSnowflakeClient.select(@db_pointer, query) { |row| @logger.info("#{row[0]} #{row[1]}") } + end + + # @example process_unshipped_web_sales {|row| check_shipping_queue(row)} + def process_unshipped_web_sales(limit = 1_000, &block) + raise('db not connected') unless @db_pointer + query = <<~QUERY.freeze + select c_first_name, c_last_name, ws_sold_date_sk, ws_list_price + from "CUSTOMER" + inner join "WEB_SALES" + ON c_customer_sk = ws_bill_customer_sk + where ws_ship_date_sk is null + #{"limit #{limit}" if limit} + QUERY + + GoSnowflakeClient.select(@db_pointer, query, &block) + end + + def log_error + @logger ||= Logger.new(STDERR) + @logger.error(GoSnowflakeClient.last_error) + end +end diff --git a/ext/ruby_snowflake.go b/ext/ruby_snowflake.go index 63d8103..d6a0d85 100644 --- a/ext/ruby_snowflake.go +++ b/ext/ruby_snowflake.go @@ -5,13 +5,13 @@ package main */ import "C" import ( - "database/sql" - "errors" - sf "github.com/snowflakedb/gosnowflake" - "unsafe" - "io" - gopointer "github.com/mattn/go-pointer" -// "fmt" + "database/sql" + "errors" + gopointer "github.com/mattn/go-pointer" + sf "github.com/snowflakedb/gosnowflake" + "io" + "unsafe" + // "fmt" ) // Lazy coding: storing last error and connection as global vars bc don't want to figure out how to pkg and pass them @@ -20,139 +20,187 @@ var last_error error //export LastError func LastError() *C.char { - if last_error == nil { - return nil - } else { - return C.CString(last_error.Error()) - } + if last_error == nil { + return nil + } else { + return C.CString(last_error.Error()) + } } // @returns db pointer // ugh, ruby and go were disagreeing about the length of `int` so I had to be particular here and in the ffi //export Connect func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.char, - user *C.char, password *C.char, role *C.char, port int64) unsafe.Pointer { - // other optional parms: Application, Host, and alt auth schemes - cfg := &sf.Config{ - Account: C.GoString(account), - Warehouse: C.GoString(warehouse), - Database: C.GoString(database), - Schema: C.GoString(schema), - User: C.GoString(user), - Password: C.GoString(password), - Role: C.GoString(role), - Region: "us-east-1", - Port: int(port), - } - - dsn, last_error := sf.DSN(cfg) - if last_error != nil { - return nil - } - - var db *sql.DB - db, last_error = sql.Open("snowflake", dsn) - if db == nil { - return nil - } else { - return gopointer.Save(db) - } + user *C.char, password *C.char, role *C.char, port int64) unsafe.Pointer { + // other optional parms: Application, Host, and alt auth schemes + cfg := &sf.Config{ + Account: C.GoString(account), + Warehouse: C.GoString(warehouse), + Database: C.GoString(database), + Schema: C.GoString(schema), + User: C.GoString(user), + Password: C.GoString(password), + Role: C.GoString(role), + Region: "us-east-1", + Port: int(port), + } + + dsn, last_error := sf.DSN(cfg) + if last_error != nil { + return nil + } + + var db *sql.DB + db, last_error = sql.Open("snowflake", dsn) + if db == nil { + return nil + } else { + return gopointer.Save(db) + } } //export Close func Close(db_pointer unsafe.Pointer) { - db := decodeDbPointer(db_pointer) - if db != nil { - db.Close() - } + db := decodeDbPointer(db_pointer) + if db != nil { + db.Close() + } } // @return number of rows affected or -1 for error //export Exec func Exec(db_pointer unsafe.Pointer, statement *C.char) int64 { - db := decodeDbPointer(db_pointer) - var res sql.Result - res, last_error = db.Exec(C.GoString(statement)) - if res != nil { - rows, _ := res.RowsAffected() - return rows - } - return -1 + db := decodeDbPointer(db_pointer) + var res sql.Result + res, last_error = db.Exec(C.GoString(statement)) + if res != nil { + rows, _ := res.RowsAffected() + return rows + } + return -1 } //export Fetch func Fetch(db_pointer unsafe.Pointer, statement *C.char) unsafe.Pointer { - db := decodeDbPointer(db_pointer) - var rows *sql.Rows - rows, last_error = db.Query(C.GoString(statement)) - if rows != nil { - result := gopointer.Save(rows) - return result - } else { - return nil - } + db := decodeDbPointer(db_pointer) + var rows *sql.Rows + rows, last_error = db.Query(C.GoString(statement)) + if rows != nil { + result := gopointer.Save(rows) + return result + } else { + return nil + } +} + +// @return column names[List] for the given query. +//export QueryColumns +func QueryColumns(rows_pointer unsafe.Pointer) **C.char { + rows := decodeRowsPointer(rows_pointer) + if rows == nil { + return nil + } + + columns, _ := rows.Columns() + rowLength := len(columns) + + // See `NextRow` for why this pattern + pointerSize := unsafe.Sizeof(rows_pointer) + // Allocate an array for the string pointers. + var out **C.char + out = (**C.char)(C.malloc(C.ulong(rowLength) * C.ulong(pointerSize))) + + pointer := out + for _, raw := range columns { + // Find where to store the address of the next string. + // Copy each output string to a C string, and add it to the array. + // C.CString uses malloc to allocate memory. + *pointer = C.CString(string(raw)) + // inc pointer to next array ele + pointer = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(pointer)) + pointerSize)) + } + return out +} + +// @return column names[List] for the given query. +//export QueryColumnCount +func QueryColumnCount(rows_pointer unsafe.Pointer) int32 { + rows := decodeRowsPointer(rows_pointer) + if rows == nil { + return 0 + } + + columns, _ := rows.Columns() + return int32(len(columns)) } // NOTE: gc's the rows_pointer object on EOF and returns nil. LastError is set to EOF //export NextRow func NextRow(rows_pointer unsafe.Pointer) **C.char { - if rows_pointer == nil { - last_error = errors.New("rows_pointer null: cannot fetch") - return nil - } - var rows *sql.Rows - rows = gopointer.Restore(rows_pointer).(*sql.Rows) - - if rows == nil { - last_error = errors.New("rows_pointer invalid: Restore returned nil") - return nil - } - - if rows.Next() { - columns, _ := rows.Columns() - rowLength := len(columns) - - rawResult := make([][]byte, rowLength) - rawData := make([]interface{}, rowLength) - for i, _ := range rawResult { // found in stackoverflow, fwiw - rawData[i] = &rawResult[i] // Put pointers to each string in the interface slice - } - - // https://stackoverflow.com/questions/58866962/how-to-pass-an-array-of-strings-and-get-an-array-of-strings-in-ruby-using-go-sha - pointerSize := unsafe.Sizeof(rows_pointer) - // Allocate an array for the string pointers. - var out **C.char - out = (**C.char)(C.malloc(C.ulong(rowLength) * C.ulong(pointerSize))) - - last_error = rows.Scan(rawData...) - if last_error != nil { - return nil - } - pointer := out - for _, raw := range rawResult { - // Find where to store the address of the next string. - // Copy each output string to a C string, and add it to the array. - // C.CString uses malloc to allocate memory. - if raw == nil { - *pointer = nil - } else { - *pointer = C.CString(string(raw)) - } - pointer = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(pointer)) + pointerSize)) - } - return out - } else if rows.Err() == io.EOF { - gopointer.Unref(rows_pointer) // free up for gc - } - return nil + rows := decodeRowsPointer(rows_pointer) + if rows == nil { + return nil + } + + if rows.Next() { + columns, _ := rows.Columns() + rowLength := len(columns) + + rawResult := make([][]byte, rowLength) + rawData := make([]interface{}, rowLength) + for i, _ := range rawResult { // found in stackoverflow, fwiw + rawData[i] = &rawResult[i] // Put pointers to each string in the interface slice + } + + // https://stackoverflow.com/questions/58866962/how-to-pass-an-array-of-strings-and-get-an-array-of-strings-in-ruby-using-go-sha + pointerSize := unsafe.Sizeof(rows_pointer) + // Allocate an array for the string pointers. + var out **C.char + out = (**C.char)(C.malloc(C.ulong(rowLength) * C.ulong(pointerSize))) + + last_error = rows.Scan(rawData...) + if last_error != nil { + return nil + } + pointer := out + for _, raw := range rawResult { + // Find where to store the address of the next string. + // Copy each output string to a C string, and add it to the array. + // C.CString uses malloc to allocate memory. + if raw == nil { + *pointer = nil + } else { + *pointer = C.CString(string(raw)) + } + pointer = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(pointer)) + pointerSize)) + } + return out + } else if rows.Err() == io.EOF { + gopointer.Unref(rows_pointer) // free up for gc + } + return nil } func decodeDbPointer(db_pointer unsafe.Pointer) *sql.DB { - if db_pointer == nil { - last_error = errors.New("db_pointer is null. Cannot process command.") - return nil - } - return gopointer.Restore(db_pointer).(*sql.DB) + if db_pointer == nil { + last_error = errors.New("db_pointer is null. Cannot process command.") + return nil + } + return gopointer.Restore(db_pointer).(*sql.DB) +} + +func decodeRowsPointer(rows_pointer unsafe.Pointer) *sql.Rows { + if rows_pointer == nil { + last_error = errors.New("rows_pointer null: cannot fetch") + return nil + } + var rows *sql.Rows + rows = gopointer.Restore(rows_pointer).(*sql.Rows) + + if rows == nil { + last_error = errors.New("rows_pointer invalid: Restore returned nil") + } + return rows } -func main(){} +func main() {} diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index 6f54846..653c694 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -35,6 +35,27 @@ def exec(db_pointer, statement) count >= 0 ? count : nil end + # Send a query and then yield each row as an array of strings to the given block + # @param db_pointer[Pointer] the pointer which `connect` returned. + # @param query[String] a select query to run. + # @return error_string + # @yield List + def select(db_pointer, sql, field_count: nil) + return nil unless db_pointer + return to_enum(__method__, db_pointer, sql) unless block_given? + + query_pointer = fetch(db_pointer, sql) + return nil if query_pointer.nil? || query_pointer == FFI::Pointer::NULL + + field_count ||= column_count(query_pointer) + loop do + row = get_next_row(query_pointer, field_count) + return last_error unless row + + yield row + end + end + # @param db_pointer[Pointer] the pointer which `connect` returned. # @param query[String] a select query to run. # @return query_object[Pointer] a pointer to use for subsequent calls not inspectable nor viewable by Ruby; however, @@ -45,6 +66,9 @@ def fetch(db_pointer, query) # @param query_object[Pointer] the pointer which `fetch` returned. Go will gc this object when the query is done; so, # don't expect to reference it after the call which returned `nil` + # @param field_count[Integer] column count: it will seg fault if you provide a number greater than the actual number. + # Using code should use wrap this in something like + # # @return [List] the column values in order def get_next_row(query_object, field_count) raw_row = GoSnowflakeClientBinding.next_row(query_object) @@ -63,6 +87,30 @@ def get_next_row(query_object, field_count) LibC.free(raw_row) if raw_row end + # @param query_object[Pointer] the pointer which `fetch` returned. + # @return [List] the column values in order + def column_names(query_object, field_count = nil) + raw_row = GoSnowflakeClientBinding.query_columns(query_object) + return nil if raw_row.nil? || raw_row == FFI::Pointer::NULL + + raw_row.get_array_of_pointer(0, field_count).map do |cstr| + if cstr == FFI::Pointer::NULL || cstr.nil? + nil + else + str = cstr.read_string + LibC.free(cstr) + str + end + end + ensure + LibC.free(raw_row) if raw_row + end + + # @param query_object[Pointer] the pointer which `fetch` returned. + def column_count(query_object) + GoSnowflakeClientBinding.query_column_count(query_object) + end + # TODO write query method which takes block and iterates with an ensure to tell go to release query_object and that # takes a list of converters for casting strings to intended types @@ -86,5 +134,7 @@ module GoSnowflakeClientBinding attach_function(:exec, 'Exec', [:pointer, :string], :int64) attach_function(:fetch, 'Fetch', [:pointer, :string], :pointer) attach_function(:next_row, 'NextRow', [:pointer], :pointer) + attach_function(:query_columns, 'QueryColumns', [:pointer], :pointer) + attach_function(:query_column_count, 'QueryColumnCount', [:pointer], :int32) end end diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb index 83be6df..9480b89 100644 --- a/lib/ruby_snowflake_client/version.rb +++ b/lib/ruby_snowflake_client/version.rb @@ -1,3 +1,3 @@ module GoSnowflakeClient - VERSION = "0.2.2" + VERSION = "0.2.3" end From 9973837cf1a67c89011ff9687c5cf0da4eb2c41f Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 15 Jan 2020 13:27:49 -0500 Subject: [PATCH 09/13] CRUD examples --- examples/common_sample_interface.rb | 31 +++++++++++++++++++ examples/snowflake_sample_data.rb | 28 ++---------------- examples/table_crud.rb | 46 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 examples/common_sample_interface.rb create mode 100644 examples/table_crud.rb diff --git a/examples/common_sample_interface.rb b/examples/common_sample_interface.rb new file mode 100644 index 0000000..583c3cd --- /dev/null +++ b/examples/common_sample_interface.rb @@ -0,0 +1,31 @@ +$LOAD_PATH << File.expand_path('../..', __FILE__) +require 'lib/go_snowflake_client' +require 'logger' + +class CommonSampleInterface + attr_reader :db_pointer + + def initialize(database) + @logger = Logger.new(STDERR) + + @db_pointer = GoSnowflakeClient.connect( + ENV['SNOWFLAKE_TEST_ACCOUNT'], + ENV['SNOWFLAKE_TEST_WAREHOUSE'], + database, + ENV['SNOWFLAKE_TEST_SCHEMA'] || 'TPCDS_SF10TCL', + ENV['SNOWFLAKE_TEST_USER'], + ENV['SNOWFLAKE_TEST_PASSWORD'], + ENV['SNOWFLAKE_TEST_ROLE'] || 'PUBLIC') + + log_error unless @db_pointer + end + + def close_db + GoSnowflakeClient.close(@db_pointer) if @db_pointer + end + + def log_error + @logger ||= Logger.new(STDERR) + @logger.error(GoSnowflakeClient.last_error) + end +end diff --git a/examples/snowflake_sample_data.rb b/examples/snowflake_sample_data.rb index ba31f9f..021599f 100644 --- a/examples/snowflake_sample_data.rb +++ b/examples/snowflake_sample_data.rb @@ -1,29 +1,12 @@ +require_relative 'common_sample_interface.rb' # Creates/uses test_data table in the db you point to # Assumes you have access to snowflake_sample_data https://docs.snowflake.net/manuals/user-guide/sample-data.html # Set env vars: SNOWFLAKE_TEST_ACCOUNT, SNOWFLAKE_TEST_USER, SNOWFLAKE_TEST_PASSWORD, SNOWFLAKE_TEST_WAREHOUSE # optionally set SNOWFLAKE_TEST_SCHEMA, SNOWFLAKE_TEST_ROLE -$LOAD_PATH << File.expand_path('../..', __FILE__) -require 'lib/go_snowflake_client' -require 'logger' -class SnowflakeSampleData +class SnowflakeSampleData < CommonSampleInterface def initialize - @logger = Logger.new(STDERR) - - @db_pointer = GoSnowflakeClient.connect( - ENV['SNOWFLAKE_TEST_ACCOUNT'], - ENV['SNOWFLAKE_TEST_WAREHOUSE'], - "SNOWFLAKE_SAMPLE_DATA", - ENV['SNOWFLAKE_TEST_SCHEMA'] || 'TPCDS_SF10TCL', - ENV['SNOWFLAKE_TEST_USER'], - ENV['SNOWFLAKE_TEST_PASSWORD'], - ENV['SNOWFLAKE_TEST_ROLE'] || 'PUBLIC') - - log_error unless @db_pointer - end - - def close_db - GoSnowflakeClient.close(@db_pointer) if @db_pointer + super("SNOWFLAKE_SAMPLE_DATA") end def get_customer_names(where = "c_last_name = 'Flowers'") @@ -49,9 +32,4 @@ def process_unshipped_web_sales(limit = 1_000, &block) GoSnowflakeClient.select(@db_pointer, query, &block) end - - def log_error - @logger ||= Logger.new(STDERR) - @logger.error(GoSnowflakeClient.last_error) - end end diff --git a/examples/table_crud.rb b/examples/table_crud.rb new file mode 100644 index 0000000..733e8ee --- /dev/null +++ b/examples/table_crud.rb @@ -0,0 +1,46 @@ +require_relative 'common_sample_interface.rb' # Creates/uses test_data table in the db you point to +# Set env vars: SNOWFLAKE_TEST_ACCOUNT, SNOWFLAKE_TEST_USER, SNOWFLAKE_TEST_PASSWORD, SNOWFLAKE_TEST_WAREHOUSE, SNOWFLAKE_TEST_DATABASE +# optionally set SNOWFLAKE_TEST_SCHEMA, SNOWFLAKE_TEST_ROLE +# use GoSnowflakeClient.select(c.db_pointer, 'select * from test_table', field_count: 3).to_a to see the db contents +class TableCRUD < CommonSampleInterface + + TEST_TABLE_NAME = 'TEST_TABLE' + + def initialize + super(ENV['SNOWFLAKE_TEST_DATABASE']) + end + + def create_test_table + command = <<~COMMAND + CREATE TEMP TABLE IF NOT EXISTS #{TEST_TABLE_NAME} + (id int AUTOINCREMENT NOT NULL, + some_timestamp TIMESTAMP_TZ DEFAULT CURRENT_TIMESTAMP(), + a_string string(20)) + COMMAND + result = GoSnowflakeClient.exec(@db_pointer, command) + result || log_error + end + + # @example insert_test_table([['2019-07-04 04:12:31 +0000', 'foo'],['2019-07-04 04:12:31 -0600', 'bar'],[Time.now, 'quux']]) + def insert_test_table(time_string_pairs) + command = <<~COMMAND + INSERT INTO #{TEST_TABLE_NAME} (some_timestamp, a_string) + VALUES #{time_string_pairs.map {|time, text| "('#{time}', '#{text}')"}.join(', ')} + COMMAND + result = GoSnowflakeClient.exec(@db_pointer, command) + result || log_error + end + + # @example update_test_table([[1, 'foo'],[99, 'bar'],[31, 'quux']]) + def update_test_table(id_string_pairs) + id_string_pairs.map do |id, text| + command = <<~COMMAND + UPDATE #{TEST_TABLE_NAME} + SET a_string = '#{text}' + WHERE id = #{id} + COMMAND + result = GoSnowflakeClient.exec(@db_pointer, command) + result || log_error + end + end +end From 4bfc34c5d7f1f0acce2a40b7a109f0fb8d5435d0 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 15 Jan 2020 14:33:22 -0500 Subject: [PATCH 10/13] Filled in README --- README.md | 92 ++++++++++++++++++++++++++++++++++++++ lib/go_snowflake_client.rb | 8 ++-- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e69de29..a6f27e2 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,92 @@ +# Snowflake Connector for Ruby + +Uses [gosnowflake](https://github.com/snowflakedb/gosnowflake/) to more efficiently query snowflake than ODBC. We found +at least 2 significant problems with ODBC which this resolves: +1. For large result sets, ODBC would get progressively slower per row as it would retrieve all the preceding +pages in order to figure out the offset. This new gem uses a streaming interface alleviating the need for +offsets and limit when paging through result sets. +2. ODBC mangled timezone information. + +In addition, this gem is a lot faster for all but the most trivial queries. + +## Tech Overview + +This gem works by deserializing each row into an array of strings in Go. It then converts it to an array +of C strings (`**C.Char`) which it passes back through the FFI (foreign function interface) to Ruby. +There's a slight penalty for the 4 time type conversion (from the db type to Go string, from Go string +to C string, from C string to the Ruby string, and then from Ruby string to your intended type). + +## How to use + +Look at [examples](https://github.com/dmitchell/go-ruby-snowflake-connector/blob/master/examples) + +1. add as gem to your project (`gem 'ruby_snowflake_client', '~> 0.2.2'`) +2. put `require 'go_snowflake_client'` at the top of your files which use it +3. following the pattern of the [example connect](https://github.com/dmitchell/go-ruby-snowflake-connector/blob/master/examples/table_crud.rb), +call `GoSnowflakeClient.connect` with your database information and credentials. +4. use `GoSnowflakeClient.exec` to execute create, update, delete, and insert queries. If it +returns `nil`, call `GoSnowflakeClient.last_error` to get the error. Otherwise, it will return +the number of affected rows. +5. use `GoSnowflakeClient.select` with a block to execute on each row to query the database. This +will return either `nil` or an error string. +9. and finally, call `GoSnowflakeClient.close(db_pointer)` to close the database connection + +### Our use pattern + +In our application, we've wrapped this library with query generators and model definitions somewhat ala +Rails but with less dynamic introspection although we could add it by using +``` ruby +GoSnowflakeClient.select(db, 'describe table my_table') do |col_name, col_type, _, nullable, *_| + my_table.add_column_description(col_name, col_type, nullable) +end +``` + +Each snowflake model class inherits from an abstract class which instantiates model instances +from each query by a pattern like +``` ruby + GoSnowflakeClient.select(db, query) do |row| + entry = self.new(fields.zip(row).map {|field, value| cast(field, value)}.to_h) + yield entry + end + + def cast(field_name, value) + if value.nil? + [field_name, value] + elsif column_name_to_cast.include?(field_name) + cast_method = column_name_to_cast[field_name] + if cast_method == :to_time + [field_name, value.to_time(:local)] + elsif cast_method == :to_utc + [field_name, value.to_time(:utc)] + elsif cast_method == :to_date + [field_name, value.to_date] + elsif cast_method == :to_utc_date + [field_name, value.to_time(:utc).to_date] + else + [field_name, value.public_send(cast_method)] + end + else + [field_name, value] + end + end + +# where each model declares column_name_to_cast ala + COLUMN_NAME_TO_CAST = { + id: :to_i, + ad_text_id: :to_i, + is_mobile: :to_bool, + is_full_site: :to_bool, + action_element_count: :to_i, + created_at: :to_time, + session_idx: :to_i, + log_idx: :to_i, + log_date: :to_utc_date, + log_path_date: :to_utc_date}.with_indifferent_access.freeze + + def self.column_name_to_cast + COLUMN_NAME_TO_CAST + end +``` + +Of course, instantiating an object for each row adds expense and gc stress; so, it may not always +be a good approach. diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index 653c694..1a62b71 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -41,11 +41,11 @@ def exec(db_pointer, statement) # @return error_string # @yield List def select(db_pointer, sql, field_count: nil) - return nil unless db_pointer + return "db_pointer not initialized" unless db_pointer return to_enum(__method__, db_pointer, sql) unless block_given? query_pointer = fetch(db_pointer, sql) - return nil if query_pointer.nil? || query_pointer == FFI::Pointer::NULL + return last_error if query_pointer.nil? || query_pointer == FFI::Pointer::NULL field_count ||= column_count(query_pointer) loop do @@ -54,6 +54,7 @@ def select(db_pointer, sql, field_count: nil) yield row end + nil end # @param db_pointer[Pointer] the pointer which `connect` returned. @@ -111,9 +112,6 @@ def column_count(query_object) GoSnowflakeClientBinding.query_column_count(query_object) end - # TODO write query method which takes block and iterates with an ensure to tell go to release query_object and that - # takes a list of converters for casting strings to intended types - module LibC extend FFI::Library ffi_lib(FFI::Library::LIBC) From 862b88a68ff11b48d2241822f6516181c248fd96 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Wed, 15 Jan 2020 14:46:25 -0500 Subject: [PATCH 11/13] Don't hardcode region --- README.md | 3 +-- ext/ruby_snowflake.go | 1 - lib/go_snowflake_client.rb | 4 ++-- lib/ruby_snowflake_client/version.rb | 2 +- ruby_snowflake_client.gemspec | 26 -------------------------- 5 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 ruby_snowflake_client.gemspec diff --git a/README.md b/README.md index a6f27e2..d2760d5 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,7 @@ from each query by a pattern like created_at: :to_time, session_idx: :to_i, log_idx: :to_i, - log_date: :to_utc_date, - log_path_date: :to_utc_date}.with_indifferent_access.freeze + log_date: :to_utc_date}.with_indifferent_access.freeze def self.column_name_to_cast COLUMN_NAME_TO_CAST diff --git a/ext/ruby_snowflake.go b/ext/ruby_snowflake.go index d6a0d85..647f030 100644 --- a/ext/ruby_snowflake.go +++ b/ext/ruby_snowflake.go @@ -41,7 +41,6 @@ func Connect(account *C.char, warehouse *C.char, database *C.char, schema *C.cha User: C.GoString(user), Password: C.GoString(password), Role: C.GoString(role), - Region: "us-east-1", Port: int(port), } diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index 1a62b71..88f7bdc 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -2,7 +2,7 @@ require 'ruby_snowflake_client/version' require 'ffi' -# Note: this library is not thread safe as it caches the db and last error +# Note: this library is not thread safe as it caches the last error # The call pattern expectation is to call last_error after any call which may have gotten an error. If last_error is # `nil`, there was no error. module GoSnowflakeClient @@ -15,7 +15,7 @@ def last_error() error end - # @param account[String] should include everything in the db url ahead of region.snowflakecomputing.com + # @param account[String] should include everything in the db url ahead of 'snowflakecomputing.com' # @param port[Integer] # @return query_object[Pointer] a pointer to use for subsequent calls not inspectable nor viewable by Ruby def connect(account, warehouse, database, schema, user, password, role, port = 443) diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb index 9480b89..f3fc14c 100644 --- a/lib/ruby_snowflake_client/version.rb +++ b/lib/ruby_snowflake_client/version.rb @@ -1,3 +1,3 @@ module GoSnowflakeClient - VERSION = "0.2.3" + VERSION = "0.2.4" end diff --git a/ruby_snowflake_client.gemspec b/ruby_snowflake_client.gemspec deleted file mode 100644 index c1d707c..0000000 --- a/ruby_snowflake_client.gemspec +++ /dev/null @@ -1,26 +0,0 @@ -lib = File.expand_path('../lib', __FILE__) -$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'ruby_snowflake_client/version' - -Gem::Specification.new do |s| - s.name = "ruby_snowflake_client" - s.version = GoSnowflakeClient::VERSION - s.summary = "Snowflake connect for Ruby" - s.author = "CarGurus" - s.email = ['dmitchell@cargurus.com', 'sabbott@cargurus.com'] - s.platform = Gem::Platform::CURRENT - s.description = <<~DESC - Uses gosnowflake to connect to and communicate with Snowflake. - This library is much faster than using ODBC especially for large result sets and avoids ODBC butchering of timezones. - DESC - s.license = 'MIT' # TODO double check - - s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb', 'lib/ruby_snowflake_client/version.rb'] - - # perhaps nothing and figure out how to build and pkg the platform specific .so, or .a, or ... - #s.extensions << "ext/ruby_snowflake_client/extconf.rb" - - s.add_dependency 'ffi' - s.add_development_dependency "bundler" - s.add_development_dependency "rake" -end From 23c2547dbe455f01490adbbd527ee09780f826ef Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Thu, 16 Jan 2020 13:27:34 -0500 Subject: [PATCH 12/13] Restore gemspec --- ruby_snowflake_client.gemspec | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ruby_snowflake_client.gemspec diff --git a/ruby_snowflake_client.gemspec b/ruby_snowflake_client.gemspec new file mode 100644 index 0000000..c1d707c --- /dev/null +++ b/ruby_snowflake_client.gemspec @@ -0,0 +1,26 @@ +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'ruby_snowflake_client/version' + +Gem::Specification.new do |s| + s.name = "ruby_snowflake_client" + s.version = GoSnowflakeClient::VERSION + s.summary = "Snowflake connect for Ruby" + s.author = "CarGurus" + s.email = ['dmitchell@cargurus.com', 'sabbott@cargurus.com'] + s.platform = Gem::Platform::CURRENT + s.description = <<~DESC + Uses gosnowflake to connect to and communicate with Snowflake. + This library is much faster than using ODBC especially for large result sets and avoids ODBC butchering of timezones. + DESC + s.license = 'MIT' # TODO double check + + s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb', 'lib/ruby_snowflake_client/version.rb'] + + # perhaps nothing and figure out how to build and pkg the platform specific .so, or .a, or ... + #s.extensions << "ext/ruby_snowflake_client/extconf.rb" + + s.add_dependency 'ffi' + s.add_development_dependency "bundler" + s.add_development_dependency "rake" +end From 6415690f0f16e8036c518286417afbd0e1fe72ed Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Thu, 16 Jan 2020 14:26:44 -0500 Subject: [PATCH 13/13] rubocop changes --- Gemfile | 2 ++ Gemfile.lock | 2 +- Rakefile | 4 +++- examples/common_sample_interface.rb | 7 +++++-- examples/snowflake_sample_data.rb | 12 +++++++----- examples/table_crud.rb | 7 ++++--- lib/go_snowflake_client.rb | 18 ++++++++++-------- lib/ruby_snowflake_client/version.rb | 4 +++- ruby_snowflake_client.gemspec | 23 ++++++++++++++--------- 9 files changed, 49 insertions(+), 30 deletions(-) diff --git a/Gemfile b/Gemfile index 396c3dc..0748779 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,5 @@ +# frozen_string_literal: true + source 'https://rubygems.org' # Specify your gem's dependencies in scatter.gemspec diff --git a/Gemfile.lock b/Gemfile.lock index e897a21..aeefa63 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ruby_snowflake_client (0.2.3-x86_64-darwin-18) + ruby_snowflake_client (0.2.4-x86_64-darwin-18) ffi GEM diff --git a/Rakefile b/Rakefile index 2995527..7398a90 100644 --- a/Rakefile +++ b/Rakefile @@ -1 +1,3 @@ -require "bundler/gem_tasks" +# frozen_string_literal: true + +require 'bundler/gem_tasks' diff --git a/examples/common_sample_interface.rb b/examples/common_sample_interface.rb index 583c3cd..84f16ee 100644 --- a/examples/common_sample_interface.rb +++ b/examples/common_sample_interface.rb @@ -1,4 +1,6 @@ -$LOAD_PATH << File.expand_path('../..', __FILE__) +# frozen_string_literal: true + +$LOAD_PATH << File.expand_path('..', __dir__) require 'lib/go_snowflake_client' require 'logger' @@ -15,7 +17,8 @@ def initialize(database) ENV['SNOWFLAKE_TEST_SCHEMA'] || 'TPCDS_SF10TCL', ENV['SNOWFLAKE_TEST_USER'], ENV['SNOWFLAKE_TEST_PASSWORD'], - ENV['SNOWFLAKE_TEST_ROLE'] || 'PUBLIC') + ENV['SNOWFLAKE_TEST_ROLE'] || 'PUBLIC' + ) log_error unless @db_pointer end diff --git a/examples/snowflake_sample_data.rb b/examples/snowflake_sample_data.rb index 021599f..0ba6b8e 100644 --- a/examples/snowflake_sample_data.rb +++ b/examples/snowflake_sample_data.rb @@ -1,18 +1,19 @@ +# frozen_string_literal: true + require_relative 'common_sample_interface.rb' # Creates/uses test_data table in the db you point to # Assumes you have access to snowflake_sample_data https://docs.snowflake.net/manuals/user-guide/sample-data.html # Set env vars: SNOWFLAKE_TEST_ACCOUNT, SNOWFLAKE_TEST_USER, SNOWFLAKE_TEST_PASSWORD, SNOWFLAKE_TEST_WAREHOUSE # optionally set SNOWFLAKE_TEST_SCHEMA, SNOWFLAKE_TEST_ROLE class SnowflakeSampleData < CommonSampleInterface - def initialize - super("SNOWFLAKE_SAMPLE_DATA") + super('SNOWFLAKE_SAMPLE_DATA') end def get_customer_names(where = "c_last_name = 'Flowers'") raise('db not connected') unless @db_pointer - query = "select c_first_name, c_last_name from \"CUSTOMER\"" + query = 'select c_first_name, c_last_name from "CUSTOMER"' query += " where #{where}" if where GoSnowflakeClient.select(@db_pointer, query) { |row| @logger.info("#{row[0]} #{row[1]}") } @@ -21,9 +22,10 @@ def get_customer_names(where = "c_last_name = 'Flowers'") # @example process_unshipped_web_sales {|row| check_shipping_queue(row)} def process_unshipped_web_sales(limit = 1_000, &block) raise('db not connected') unless @db_pointer - query = <<~QUERY.freeze + + query = <<~QUERY select c_first_name, c_last_name, ws_sold_date_sk, ws_list_price - from "CUSTOMER" + from "CUSTOMER" inner join "WEB_SALES" ON c_customer_sk = ws_bill_customer_sk where ws_ship_date_sk is null diff --git a/examples/table_crud.rb b/examples/table_crud.rb index 733e8ee..3774a2d 100644 --- a/examples/table_crud.rb +++ b/examples/table_crud.rb @@ -1,9 +1,10 @@ +# frozen_string_literal: true + require_relative 'common_sample_interface.rb' # Creates/uses test_data table in the db you point to # Set env vars: SNOWFLAKE_TEST_ACCOUNT, SNOWFLAKE_TEST_USER, SNOWFLAKE_TEST_PASSWORD, SNOWFLAKE_TEST_WAREHOUSE, SNOWFLAKE_TEST_DATABASE # optionally set SNOWFLAKE_TEST_SCHEMA, SNOWFLAKE_TEST_ROLE # use GoSnowflakeClient.select(c.db_pointer, 'select * from test_table', field_count: 3).to_a to see the db contents class TableCRUD < CommonSampleInterface - TEST_TABLE_NAME = 'TEST_TABLE' def initialize @@ -25,7 +26,7 @@ def create_test_table def insert_test_table(time_string_pairs) command = <<~COMMAND INSERT INTO #{TEST_TABLE_NAME} (some_timestamp, a_string) - VALUES #{time_string_pairs.map {|time, text| "('#{time}', '#{text}')"}.join(', ')} + VALUES #{time_string_pairs.map { |time, text| "('#{time}', '#{text}')" }.join(', ')} COMMAND result = GoSnowflakeClient.exec(@db_pointer, command) result || log_error @@ -35,7 +36,7 @@ def insert_test_table(time_string_pairs) def update_test_table(id_string_pairs) id_string_pairs.map do |id, text| command = <<~COMMAND - UPDATE #{TEST_TABLE_NAME} + UPDATE #{TEST_TABLE_NAME} SET a_string = '#{text}' WHERE id = #{id} COMMAND diff --git a/lib/go_snowflake_client.rb b/lib/go_snowflake_client.rb index 88f7bdc..c74753a 100644 --- a/lib/go_snowflake_client.rb +++ b/lib/go_snowflake_client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + $LOAD_PATH << File.dirname(__FILE__) require 'ruby_snowflake_client/version' require 'ffi' @@ -6,10 +8,10 @@ # The call pattern expectation is to call last_error after any call which may have gotten an error. If last_error is # `nil`, there was no error. module GoSnowflakeClient - extend self + module_function # @return String last error or nil. May be end of file which is not really an error - def last_error() + def last_error error, cptr = GoSnowflakeClientBinding.last_error LibC.free(cptr) if error error @@ -31,7 +33,7 @@ def close(db_pointer) # @param statement[String] an executable query which should return number of rows affected # @return rowcount[Number] number of rows or nil if there was an error def exec(db_pointer, statement) - count = GoSnowflakeClientBinding.exec(db_pointer, statement) # returns -1 for error + count = GoSnowflakeClientBinding.exec(db_pointer, statement) # returns -1 for error count >= 0 ? count : nil end @@ -41,7 +43,7 @@ def exec(db_pointer, statement) # @return error_string # @yield List def select(db_pointer, sql, field_count: nil) - return "db_pointer not initialized" unless db_pointer + return 'db_pointer not initialized' unless db_pointer return to_enum(__method__, db_pointer, sql) unless block_given? query_pointer = fetch(db_pointer, sql) @@ -124,13 +126,13 @@ module GoSnowflakeClientBinding POINTER_SIZE = FFI.type_size(:pointer) - ffi_lib(File.expand_path('../ext/ruby_snowflake_client.so', File.dirname(__FILE__))) + ffi_lib(File.expand_path('../ext/ruby_snowflake_client.so', __dir__)) attach_function(:last_error, 'LastError', [], :strptr) # ugh, `port` in gosnowflake is just :int; however, ruby - ffi -> go is passing 32bit int if I just decl :int. - attach_function(:connect, 'Connect', [:string, :string, :string, :string, :string, :string, :string, :int64], :pointer) + attach_function(:connect, 'Connect', %i[string string string string string string string int64], :pointer) attach_function(:close, 'Close', [:pointer], :void) - attach_function(:exec, 'Exec', [:pointer, :string], :int64) - attach_function(:fetch, 'Fetch', [:pointer, :string], :pointer) + attach_function(:exec, 'Exec', %i[pointer string], :int64) + attach_function(:fetch, 'Fetch', %i[pointer string], :pointer) attach_function(:next_row, 'NextRow', [:pointer], :pointer) attach_function(:query_columns, 'QueryColumns', [:pointer], :pointer) attach_function(:query_column_count, 'QueryColumnCount', [:pointer], :int32) diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb index f3fc14c..76c17d8 100644 --- a/lib/ruby_snowflake_client/version.rb +++ b/lib/ruby_snowflake_client/version.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module GoSnowflakeClient - VERSION = "0.2.4" + VERSION = '0.2.4' end diff --git a/ruby_snowflake_client.gemspec b/ruby_snowflake_client.gemspec index c1d707c..ecda7b1 100644 --- a/ruby_snowflake_client.gemspec +++ b/ruby_snowflake_client.gemspec @@ -1,26 +1,31 @@ -lib = File.expand_path('../lib', __FILE__) +# frozen_string_literal: true + +lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'ruby_snowflake_client/version' Gem::Specification.new do |s| - s.name = "ruby_snowflake_client" + s.name = 'ruby_snowflake_client' s.version = GoSnowflakeClient::VERSION - s.summary = "Snowflake connect for Ruby" - s.author = "CarGurus" + s.summary = 'Snowflake connect for Ruby' + s.author = 'CarGurus' s.email = ['dmitchell@cargurus.com', 'sabbott@cargurus.com'] s.platform = Gem::Platform::CURRENT s.description = <<~DESC Uses gosnowflake to connect to and communicate with Snowflake. This library is much faster than using ODBC especially for large result sets and avoids ODBC butchering of timezones. DESC - s.license = 'MIT' # TODO double check + s.license = 'MIT' # TODO: double check - s.files = ['ext/ruby_snowflake_client.h', 'ext/ruby_snowflake_client.so', 'lib/go_snowflake_client.rb', 'lib/ruby_snowflake_client/version.rb'] + s.files = ['ext/ruby_snowflake_client.h', + 'ext/ruby_snowflake_client.so', + 'lib/go_snowflake_client.rb', + 'lib/ruby_snowflake_client/version.rb'] # perhaps nothing and figure out how to build and pkg the platform specific .so, or .a, or ... - #s.extensions << "ext/ruby_snowflake_client/extconf.rb" + # s.extensions << "ext/ruby_snowflake_client/extconf.rb" s.add_dependency 'ffi' - s.add_development_dependency "bundler" - s.add_development_dependency "rake" + s.add_development_dependency 'bundler' + s.add_development_dependency 'rake' end