From 2fe13df6ab1ab196adbf935351387e884b0b98b4 Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 6 Apr 2026 16:21:51 -1000 Subject: [PATCH 1/6] Make tokenizer a multi command binary The tokenizer command will support subcommands for displaying its version, running the server, displaying the seal key, sealing tokens, and unsealing tokens. This will allow the deployed binary to be used directly for multiple purposes without shipping seperate helpers and will simplify instructions for setting up and using the tokenizer. --- Dockerfile | 2 +- README.md | 2 +- cmd/tokenizer/README.md | 4 +- cmd/tokenizer/args.go | 33 ++++ cmd/tokenizer/debug_listener.go | 60 ++++++++ cmd/tokenizer/main.go | 257 ++++---------------------------- cmd/tokenizer/seal.go | 76 ++++++++++ cmd/tokenizer/sealkey.go | 35 +++++ cmd/tokenizer/serve.go | 136 +++++++++++++++++ cmd/tokenizer/unseal.go | 51 +++++++ cmd/tokenizer/version.go | 51 +++++++ docs/QuickStart.md | 9 +- docs/UserGuide.md | 29 ++-- tokenizer.go | 37 +++-- 14 files changed, 519 insertions(+), 263 deletions(-) create mode 100644 cmd/tokenizer/args.go create mode 100644 cmd/tokenizer/debug_listener.go create mode 100644 cmd/tokenizer/seal.go create mode 100644 cmd/tokenizer/sealkey.go create mode 100644 cmd/tokenizer/serve.go create mode 100644 cmd/tokenizer/unseal.go create mode 100644 cmd/tokenizer/version.go diff --git a/Dockerfile b/Dockerfile index e697c04..782f4af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,4 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM alpine:latest AS runner WORKDIR /root COPY --from=builder /go/src/github.com/superfly/tokenizer/bin/tokenizer /usr/local/bin/tokenizer -CMD ["tokenizer"] +CMD ["tokenizer", "serve", "-use-flysrc=true"] diff --git a/README.md b/README.md index b22bdd9..97801ba 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ export OPEN_KEY=$(openssl rand -hex 32) Run the tokenizer server: ```shell -tokenizer +tokenizer serve -use-flysrc=true ``` The output will contain the public (seal) key, which can be used for encrypting secrets. diff --git a/cmd/tokenizer/README.md b/cmd/tokenizer/README.md index dc43a42..922145f 100644 --- a/cmd/tokenizer/README.md +++ b/cmd/tokenizer/README.md @@ -15,7 +15,7 @@ To run a test server locally: source ./.envrc # run server -go run . +go run . serve ``` -The `github.com/superfly/tokenizer/cmd/curl` package has instructions for sending requests via this server with a test client. \ No newline at end of file +The `github.com/superfly/tokenizer/cmd/curl` package has instructions for sending requests via this server with a test client. diff --git a/cmd/tokenizer/args.go b/cmd/tokenizer/args.go new file mode 100644 index 0000000..6477f20 --- /dev/null +++ b/cmd/tokenizer/args.go @@ -0,0 +1,33 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" +) + +// parseFlags runs the flagset parser. +// It fails if there are any non-flag arguments. +// On error it prints the default usage and moreUsage and exits. +func parseFlags(fs *flag.FlagSet, moreUsage string, args []string) { + err := fs.Parse(args) + if err == nil && len(fs.Args()) > 0 { + err = fmt.Errorf("unknown arguments: %v\n", strings.Join(fs.Args(), " ")) + fmt.Fprintf(os.Stderr, "%v\n", err) + fs.Usage() + } + + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", moreUsage) + os.Exit(1) + } +} + +// strToBool converts a string to a boolean. If there is an error parsing the string +// it returns a default value of false. Values 1, t, T, TRUE, true, and True return true values. +func strToBool(s string) bool { + b, _ := strconv.ParseBool(s) + return b +} diff --git a/cmd/tokenizer/debug_listener.go b/cmd/tokenizer/debug_listener.go new file mode 100644 index 0000000..75340ad --- /dev/null +++ b/cmd/tokenizer/debug_listener.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "net" + "time" +) + +type debugListener struct { + net.Listener +} + +func (dl debugListener) Accept() (net.Conn, error) { + c, err := dl.Listener.Accept() + if err == nil { + c = debugConn{c} + } + return c, err +} + +type debugConn struct { + c net.Conn +} + +func (dc debugConn) Read(b []byte) (int, error) { + n, err := dc.c.Read(b) + if err == nil { + fmt.Printf("<- %#v\n", string(b[:n])) + } + return n, err +} + +func (dc debugConn) Write(b []byte) (int, error) { + fmt.Printf("-> %#v\n", string(b)) + return dc.c.Write(b) +} + +func (dc debugConn) Close() error { + return dc.c.Close() +} + +func (dc debugConn) LocalAddr() net.Addr { + return dc.c.LocalAddr() +} + +func (dc debugConn) RemoteAddr() net.Addr { + return dc.c.RemoteAddr() +} + +func (dc debugConn) SetDeadline(t time.Time) error { + return dc.c.SetDeadline(t) +} + +func (dc debugConn) SetReadDeadline(t time.Time) error { + return dc.c.SetReadDeadline(t) +} + +func (dc debugConn) SetWriteDeadline(t time.Time) error { + return dc.c.SetWriteDeadline(t) +} diff --git a/cmd/tokenizer/main.go b/cmd/tokenizer/main.go index 52ebd67..492af01 100644 --- a/cmd/tokenizer/main.go +++ b/cmd/tokenizer/main.go @@ -1,24 +1,11 @@ package main import ( - "context" - "errors" - "flag" "fmt" - "net" - "net/http" "os" - "os/signal" - "runtime/debug" "strings" - "syscall" - "time" - "github.com/sirupsen/logrus" "github.com/superfly/tokenizer" - "golang.org/x/exp/slices" - - "github.com/superfly/flysrc-go" ) // Package variables can be overridden at build time: @@ -30,11 +17,8 @@ var ( // Address for HTTP proxy to listen at. ListenAddress = ":8080" -) -var ( - versionFlag = flag.Bool("version", false, "print the version number") - sealKeyFlag = flag.Bool("sealkey", false, "print the seal key and exit") + Version = "" ) func init() { @@ -54,228 +38,43 @@ func init() { } func main() { - flag.Usage = usage - flag.Parse() - - switch { - case *versionFlag: - runVersion() - case *sealKeyFlag: - runSealKey() - default: - runServe() - } - -} - -func runServe() { - l, err := net.Listen("tcp", ListenAddress) - if err != nil { - logrus.WithError(err).Fatal("listen") - } - - if len(os.Getenv("DEBUG_TCP")) != 0 { - l = debugListener{l} - } - - key := os.Getenv("OPEN_KEY") - if key == "" { - fmt.Fprintf(os.Stderr, "missing OPEN_KEY\n") + cmd := "tokenizer" + if len(os.Args) < 2 { + usage() os.Exit(1) } - opts := []tokenizer.Option{} - - if hn := os.Getenv("TOKENIZER_HOSTNAMES"); hn != "" { - opts = append(opts, tokenizer.TokenizerHostnames(strings.Split(hn, ",")...)) - } - - if slices.Contains([]string{"1", "true"}, os.Getenv("OPEN_PROXY")) { - opts = append(opts, tokenizer.OpenProxy()) - } - - if slices.Contains([]string{"1", "true"}, os.Getenv("REQUIRE_FLY_SRC")) { - opts = append(opts, tokenizer.RequireFlySrc()) - } - - if slices.Contains([]string{"1", "true"}, os.Getenv("NO_FLY_SRC")) { - // nothing - } else { - parser, err := flysrc.New() - if err != nil { - logrus.WithError(err).Panic("Error making flysrc parser") - } - - opts = append(opts, tokenizer.WithFlysrcParser(parser)) - } - - tkz := tokenizer.NewTokenizer(key, opts...) - - if len(os.Getenv("DEBUG")) != 0 { - tkz.ProxyHttpServer.Verbose = true - tkz.ProxyHttpServer.Logger = logrus.StandardLogger() - } - - server := &http.Server{Handler: tkz} - - go func() { - if err := server.Serve(l); !errors.Is(err, http.ErrServerClosed) { - logrus.WithError(err).Fatal("listening") - } - }() - - logrus.WithFields(logrus.Fields{ - "address": ListenAddress, - "seal_key": tkz.SealKey(), - }).Info("listening") - - handleSignals(server) -} - -func handleSignals(server *http.Server) { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - sig := <-sigs - logrus.WithField("signal", sig).Info("graceful shutdown") - - done := make(chan struct{}) - go func() { - defer close(done) - if err := server.Shutdown(context.Background()); err != nil { - logrus.WithError(err).Warn("graceful shutdown") - } - }() - - select { - case <-done: - case <-sigs: - logrus.Info("immediate shutdown") - if err := server.Close(); err != nil { - logrus.WithError(err).Warn("immediate shutdown") - } - } -} - -func runSealKey() { - key := os.Getenv("OPEN_KEY") - if key == "" { - fmt.Fprintf(os.Stderr, "missing OPEN_KEY\n") + subCmd := os.Args[1] + qualCmd := cmd + " " + subCmd + shiftedArgs := os.Args[2:] + switch subCmd { + case "version": + runVersion(qualCmd, shiftedArgs) + case "serve": + runServe(qualCmd, shiftedArgs) + case "sealkey": + runSealkey(qualCmd, shiftedArgs) + case "seal": + runSeal(qualCmd, shiftedArgs) + case "unseal": + runUnseal(qualCmd, shiftedArgs) + default: + fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", subCmd) + usage() os.Exit(1) } - - tkz := tokenizer.NewTokenizer(key) - fmt.Fprintf(os.Stderr, "export SEAL_KEY=%v\n", tkz.SealKey()) -} - -var Version = "" - -func runVersion() { - fmt.Fprintln(os.Stderr, versionString()) -} - -func versionString() string { - if Version != "" { - return fmt.Sprintf("tokenizer %s", Version) - } else if bi, ok := debug.ReadBuildInfo(); ok { - var ( - commit string - modified bool - ) - for _, s := range bi.Settings { - if s.Key == "vcs.revision" { - commit = s.Value - } - if s.Key == "vcs.modified" { - modified = s.Value == "true" - } - } - - switch { - case modified: - // dev build - case bi.Main.Version != "(devel)" && commit != "": - return fmt.Sprintf("tokenizer %s, commit=%s", bi.Main.Version, commit) - case bi.Main.Version != "(devel)": - return fmt.Sprintf("tokenizer %s", bi.Main.Version) - case commit != "": - return fmt.Sprintf("tokenizer commit=%s", commit) - } - } - return "tokenizer development build" } func usage() { - fmt.Fprintf(os.Stderr, `tokenizer is an HTTP proxy that injects third party authentication credentials into requests + fmt.Fprintf(os.Stderr, ` +tokenizer is an HTTP proxy that injects third party authentication credentials into requests Usage: - tokenizer [flags] - -Flags: - -version prints the version - -help prints this message - -Configuration — tokenizer is configured using the following environment variables: - - OPEN_KEY - Hex encoded curve25519 private key. You can provide 32 - random, hex encoded bytes. The log output will contain - the associated public key. - LISTEN_ADDRESS - The host:port address to listen at. Default: ":8080" - FILTERED_HEADERS - Comma separated list of headers to filter from client - requests. + tokenizer version [flags] - show version + tokenizer serve [flags] - serve requests + tokenizer sealkey [flags] - show seal key + tokenizer seal [flags] - seal a token using the seal key + tokenizer unseal [flags] - unseal a sealed token using the open key `) } - -type debugListener struct { - net.Listener -} - -func (dl debugListener) Accept() (net.Conn, error) { - c, err := dl.Listener.Accept() - if err == nil { - c = debugConn{c} - } - return c, err -} - -type debugConn struct { - c net.Conn -} - -func (dc debugConn) Read(b []byte) (int, error) { - n, err := dc.c.Read(b) - if err == nil { - fmt.Printf("<- %#v\n", string(b[:n])) - } - return n, err -} - -func (dc debugConn) Write(b []byte) (int, error) { - fmt.Printf("-> %#v\n", string(b)) - return dc.c.Write(b) -} - -func (dc debugConn) Close() error { - return dc.c.Close() -} - -func (dc debugConn) LocalAddr() net.Addr { - return dc.c.LocalAddr() -} - -func (dc debugConn) RemoteAddr() net.Addr { - return dc.c.RemoteAddr() -} - -func (dc debugConn) SetDeadline(t time.Time) error { - return dc.c.SetDeadline(t) -} - -func (dc debugConn) SetReadDeadline(t time.Time) error { - return dc.c.SetReadDeadline(t) -} - -func (dc debugConn) SetWriteDeadline(t time.Time) error { - return dc.c.SetWriteDeadline(t) -} diff --git a/cmd/tokenizer/seal.go b/cmd/tokenizer/seal.go new file mode 100644 index 0000000..3416ff8 --- /dev/null +++ b/cmd/tokenizer/seal.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/superfly/tokenizer" +) + +var sealUsage = ` +tokenizer seal parses a json configuration for a sealed token and seals it with the the tokenizer's OPEN_KEY + +Environment variables: + + SEAL_KEY - Hex encoded sealing key, as generated by the "tokenizer sealkey" command. + DEBUG - Display debugging output when true. +` + +func runSeal(cmd string, args []string) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + var ( + sealKey = fs.String("seal-key", os.Getenv("SEAL_KEY"), "public key for sealing tokens") + jsonStr = fs.String("json", "", "JSON tokenizer secret to seal") + jsonFile = fs.String("json-file", "", "file containing JSON tokenizer secret to seal") + debug = fs.String("debug", os.Getenv("DEBUG"), "display debugging output when true") + ) + parseFlags(fs, sealUsage, args) + + if *sealKey == "" { + fmt.Fprintf(os.Stderr, "missing SEAL_KEY\n") + os.Exit(1) + } + + var j []byte + var prefix string + switch { + case (*jsonFile != "" && *jsonStr != "") || (*jsonFile == "" && *jsonStr == ""): + fmt.Fprintf(os.Stderr, "specify -json or -json-file, but not both\n") + os.Exit(1) + case *jsonFile != "": + bs, err := os.ReadFile(*jsonFile) + if err != nil { + fmt.Fprintf(os.Stderr, "%v: %v\n", *jsonFile, err) + os.Exit(1) + } + prefix = *jsonFile + ": " + j = bs + case *jsonStr != "": + prefix = "" + j = []byte(*jsonStr) + } + + var secret tokenizer.Secret + if err := json.Unmarshal([]byte(j), &secret); err != nil { + fmt.Fprintf(os.Stderr, "%s%v\n", prefix, err) + os.Exit(1) + } + + if strToBool(*debug) { + bs, err := json.Marshal(secret) + if err != nil { + fmt.Printf("error generating json for secret: %v\n", err) + } else { + fmt.Printf("json for secret: %v\n", string(bs)) + } + } + + sealed, err := secret.Seal(*sealKey) + if err != nil { + fmt.Fprintf(os.Stderr, "sealing error: %v\n", err) + os.Exit(1) + } + fmt.Printf("%v\n", sealed) +} diff --git a/cmd/tokenizer/sealkey.go b/cmd/tokenizer/sealkey.go new file mode 100644 index 0000000..b025e70 --- /dev/null +++ b/cmd/tokenizer/sealkey.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/superfly/tokenizer" +) + +var sealkeyUsage = ` +tokenizer sealkey displays the sealing key matching the tokenizer's OPEN_KEY + +Environment variables: + + OPEN_KEY - Hex encoded curve25519 private key. You can provide 32 + random, hex encoded bytes. The log output will contain + the associated public key. +` + +func runSealkey(cmd string, args []string) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + var ( + openKey = fs.String("open-key", os.Getenv("OPEN_KEY"), "private key for opening sealed tokens") + ) + parseFlags(fs, sealkeyUsage, args) + + if *openKey == "" { + fmt.Fprintf(os.Stderr, "missing OPEN_KEY\n") + os.Exit(1) + } + + tkz := tokenizer.NewTokenizer(*openKey) + fmt.Fprintf(os.Stderr, "export SEAL_KEY=%v\n", tkz.SealKey()) +} diff --git a/cmd/tokenizer/serve.go b/cmd/tokenizer/serve.go new file mode 100644 index 0000000..4baaf59 --- /dev/null +++ b/cmd/tokenizer/serve.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/superfly/tokenizer" + + "github.com/superfly/flysrc-go" +) + +var serveUsage = ` +tokenizer serve runs an HTTP proxy that injects third party authentication credentials into requests + +Environment: + + OPEN_KEY - Hex encoded curve25519 private key. You can provide 32 + random, hex encoded bytes. The log output will contain + the associated public key. + LISTEN_ADDRESS - The host:port address to listen at. Default: ":8080" + FILTERED_HEADERS - Comma separated list of headers to filter from client + requests. + OPEN_PROXY - If true, allow proxy to be used without any sealed secrets. + USE_FLYSRC - If true, use /.fly/fly-src.pub to process fly-src headers. + REQUIRE_FLYSRC - If true, reject requests that do not have a valid fly-src header. + DEBUG - If true, enables debug logging. + DEBUG_TCP - If true, enables debug logging of connections. +` + +func runServe(cmd string, args []string) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + var ( + openKey = fs.String("open-key", os.Getenv("OPEN_KEY"), "private key for opening sealed tokens") + listenAddress = fs.String("listen-address", ListenAddress, "address to listen on") + debug = fs.String("debug", os.Getenv("DEBUG"), "turn on debug logging") + debugTcp = fs.String("debug-tcp", os.Getenv("DEBUG_TCP"), "debug TCP if set to true or 1") + hostNames = fs.String("hostnames", os.Getenv("TOKENIZER_HOSTNAMES"), "hostnames assigned to the tokenizer") + openProxy = fs.String("open-proxy", os.Getenv("OPEN_PROXY"), "run an open proxy if set to true or 1") + useFlysrc = fs.String("use-flysrc", os.Getenv("USE_FLYSRC"), "support fly-src using /.fly/fly-src.pub if set to true or 1") + requireFlysrc = fs.String("require-flysrc", os.Getenv("REQUIRE_FLYSRC"), "require fly-src to use proxy if set to true or 1") + ) + parseFlags(fs, serveUsage, args) + + l, err := net.Listen("tcp", *listenAddress) + if err != nil { + logrus.WithError(err).Fatal("listen") + } + + if strToBool(*debugTcp) { + l = debugListener{l} + } + + if *openKey == "" { + fmt.Fprintf(os.Stderr, "missing OPEN_KEY\n") + os.Exit(1) + } + + opts := []tokenizer.Option{} + + if hn := *hostNames; hn != "" { + opts = append(opts, tokenizer.TokenizerHostnames(strings.Split(hn, ",")...)) + } + + if strToBool(*openProxy) { + opts = append(opts, tokenizer.OpenProxy()) + } + + if strToBool(*requireFlysrc) { + opts = append(opts, tokenizer.RequireFlySrc()) + } + + if strToBool(*useFlysrc) { + parser, err := flysrc.New() + if err != nil { + logrus.WithError(err).Panic("Error making flysrc parser") + } + + opts = append(opts, tokenizer.WithFlysrcParser(parser)) + } + + tkz := tokenizer.NewTokenizer(*openKey, opts...) + + if strToBool(*debug) { + tkz.ProxyHttpServer.Verbose = true + tkz.ProxyHttpServer.Logger = logrus.StandardLogger() + } + + server := &http.Server{Handler: tkz} + + go func() { + if err := server.Serve(l); !errors.Is(err, http.ErrServerClosed) { + logrus.WithError(err).Fatal("listening") + } + }() + + logrus.WithFields(logrus.Fields{ + "address": *listenAddress, + "seal_key": tkz.SealKey(), + }).Info("listening") + + handleSignals(server) +} + +func handleSignals(server *http.Server) { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + sig := <-sigs + logrus.WithField("signal", sig).Info("graceful shutdown") + + done := make(chan struct{}) + go func() { + defer close(done) + if err := server.Shutdown(context.Background()); err != nil { + logrus.WithError(err).Warn("graceful shutdown") + } + }() + + select { + case <-done: + case <-sigs: + logrus.Info("immediate shutdown") + if err := server.Close(); err != nil { + logrus.WithError(err).Warn("immediate shutdown") + } + } +} diff --git a/cmd/tokenizer/unseal.go b/cmd/tokenizer/unseal.go new file mode 100644 index 0000000..9dee151 --- /dev/null +++ b/cmd/tokenizer/unseal.go @@ -0,0 +1,51 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/superfly/tokenizer" +) + +var unsealUsage = ` +tokenizer unseal opens a sealed token using the tokenizer's OPEN_KEY and displays its contents in JSON format + +Environment variables: + + OPEN_KEY - Hex encoded curve25519 private key. You can provide 32 + random, hex encoded bytes. The log output will contain + the associated public key. +` + +func runUnseal(cmd string, args []string) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + var ( + openKey = fs.String("open-key", os.Getenv("OPEN_KEY"), "private key for opening sealed tokens") + token = fs.String("token", "", "sealed token") + ) + parseFlags(fs, unsealUsage, args) + + if *openKey == "" { + fmt.Fprintf(os.Stderr, "missing OPEN_KEY\n") + os.Exit(1) + } + + if *token == "" { + fmt.Fprintf(os.Stderr, "missing sealed token\n") + os.Exit(1) + } + + priv, pub, err := tokenizer.ParseOpenKey(*openKey) + if err != nil { + fmt.Fprintf(os.Stderr, "OPEN_KEY: %v\n", err) + os.Exit(1) + } + + secret, err := tokenizer.OpenSealed(*token, pub, priv) + if err != nil { + fmt.Fprintf(os.Stderr, "token: %v\n", err) + os.Exit(1) + } + fmt.Printf("%v\n", string(secret)) +} diff --git a/cmd/tokenizer/version.go b/cmd/tokenizer/version.go new file mode 100644 index 0000000..9c0645c --- /dev/null +++ b/cmd/tokenizer/version.go @@ -0,0 +1,51 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime/debug" +) + +var versionUsage = ` +tokenizer version displays the version +` + +func runVersion(cmd string, args []string) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + parseFlags(fs, versionUsage, args) + parseFlags(fs, versionUsage, args) + + fmt.Fprintln(os.Stderr, versionString()) +} + +func versionString() string { + if Version != "" { + return fmt.Sprintf("tokenizer %s", Version) + } else if bi, ok := debug.ReadBuildInfo(); ok { + var ( + commit string + modified bool + ) + for _, s := range bi.Settings { + if s.Key == "vcs.revision" { + commit = s.Value + } + if s.Key == "vcs.modified" { + modified = s.Value == "true" + } + } + + switch { + case modified: + // dev build + case bi.Main.Version != "(devel)" && commit != "": + return fmt.Sprintf("tokenizer %s, commit=%s", bi.Main.Version, commit) + case bi.Main.Version != "(devel)": + return fmt.Sprintf("tokenizer %s", bi.Main.Version) + case commit != "": + return fmt.Sprintf("tokenizer commit=%s", commit) + } + } + return "tokenizer development build" +} diff --git a/docs/QuickStart.md b/docs/QuickStart.md index 6bfb8f0..6f362ac 100644 --- a/docs/QuickStart.md +++ b/docs/QuickStart.md @@ -44,12 +44,12 @@ fly -c fly.toml.timkenizer launch # generate and set the secret "open" and "seal" keys. # install the OPEN_KEY on the server and keep the SEAL_KEY for later. export OPEN_KEY=$(openssl rand -hex 32) -export SEAL_KEY=$(go run ./cmd/tokenizer -sealkey) +export SEAL_KEY=$(go run ./cmd/tokenizer sealkey) fly -c fly.toml.timkenizer secrets set OPEN_KEY=$OPEN_KEY # use the SEAL_KEY to generate a proxy token that will inject a secret token into requests to the target. # here restricted to use against https://timflyio-go-example.fly.dev from app=thenewsh -cat > token.json <<_EOF_ +JSON=' { "inject_processor": { "token": "MY_SECRET_TOKEN" }, "allowed_hosts": ["timflyio-go-example.fly.dev"], @@ -57,9 +57,8 @@ cat > token.json <<_EOF_ "allowed_orgs": ["tim-newsham"], "allowed_apps": ["thenewsh"] } -} -_EOF_ -TOKEN=$(go run ./cmd/sealtoken -json "$(cat token.json)") +}' +TOKEN=$(go run ./cmd/tokenizer seal -json "$JSON") # install the TOKEN in your approved app and use it to access the approved url. # the secret token (MY_SECRET_TOKEN) will be added as a bearer token. diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 7465b8b..8becc14 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -5,7 +5,7 @@ This document describes how to seal tokens and how to make requests with them. # Sealing tokens To seal a token you need the sealing key of your tokenizer. -* If you have the `OPEN_KEY` set in your environment you can get the seal by running `go run ./cmd/tokenizer -sealkey`. +* If you have the `OPEN_KEY` set in your environment you can get the seal by running `go run ./cmd/tokenizer sealkey`. * The seal key is written by the server when the server is started on a line like `listening address="localhost:8080" seal_key=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`. ## Command line @@ -13,7 +13,7 @@ To seal a token you need the sealing key of your tokenizer. To seal a token with the command line * Place the sealing key in your environment as `SEAL_KEY`. * Construct the sealing information as json, perhaps putting it into a file such as `token.json`. -* Run `go run ./cmd/sealtoken -json "$(cat token.json)"` or place the sealing information directly on the command line. +* Run `go run ./cmd/tokenizer -json-file token.json` or `go run ./cmd/tokenizer -json $JSONOBJ`. Example: @@ -31,10 +31,10 @@ cat > token.json <<_EOF_ } } _EOF_ -go run ./cmd/sealtoken -json "$(cat token.json)" +go run ./cmd/tokenizer seal -json-file token.json # Using the json directly -go run ./cmd/sealtoken -json '{"inject_processor":{"token":"MY_SECRET_TOKEN"},"allowed_hosts":["timflyio-go-example.fly.dev"],"fly_src_auth":{"allowed_orgs":["tim-newsham"],"allowed_apps":["thenewsh"]}}' +go run ./cmd/tokenizer seal -json '{"inject_processor":{"token":"MY_SECRET_TOKEN"},"allowed_hosts":["timflyio-go-example.fly.dev"],"fly_src_auth":{"allowed_orgs":["tim-newsham"],"allowed_apps":["thenewsh"]}}' ``` ## Go @@ -129,7 +129,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ @@ -159,7 +159,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "bearer_auth": {"digest": "K7gNU3sdo+OL0wNhqoVWhr3g6s1xYv72ol/pe/Unols="} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ @@ -215,7 +215,7 @@ SEAL='{ ] } }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") # This will only work from Machines in the thenewsh App. echo curl -s -x https://tokenizer.fly.dev:8443 \ @@ -247,12 +247,11 @@ SEAL='{ "allowed_host_pattern": ".*\\.fly\\.dev$", "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ http://timflyio-go-example.fly.dev - ``` ## Request Processing @@ -296,7 +295,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED; {\"fmt\": \"Cower %s\"}" \ @@ -333,7 +332,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED; {\"dst\": \"Z-Auth\"}" \ @@ -369,7 +368,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ @@ -413,7 +412,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ @@ -473,7 +472,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED" \ @@ -509,7 +508,7 @@ SEAL='{ "allowed_hosts": ["timflyio-go-example.fly.dev"], "no_auth": {} }' -SEALED=$(go run cmd/sealtoken/main.go -json "$SEAL") +SEALED=$(go run ./cmd/tokenizer seal -json "$SEAL") curl -s -x https://tokenizer.fly.dev \ -H "Proxy-Tokenizer: $SEALED; {\"fmt\": \"Cower %s\", \"dst\": \"X-Auth\"}" \ diff --git a/tokenizer.go b/tokenizer.go index 86a680a..c023891 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -103,17 +103,25 @@ func TokenizerHostnames(hostnames ...string) Option { } } -func NewTokenizer(openKey string, opts ...Option) *tokenizer { +func ParseOpenKey(openKey string) (*[32]byte, *[32]byte, error) { privBytes, err := hex.DecodeString(openKey) if err != nil { - logrus.WithError(err).Panic("bad private key") + return nil, nil, fmt.Errorf("bad private key") } if len(privBytes) != 32 { - logrus.Panicf("bad private key size: %d", len(privBytes)) + fmt.Errorf("bad private key size: %d", len(privBytes)) } priv := (*[32]byte)(privBytes) pub := new([32]byte) curve25519.ScalarBaseMult(pub, priv) + return priv, pub, nil +} + +func NewTokenizer(openKey string, opts ...Option) *tokenizer { + priv, pub, err := ParseOpenKey(openKey) + if err != nil { + logrus.WithError(err).Panic(err.Error()) + } proxy := goproxy.NewProxyHttpServer() tkz := &tokenizer{ProxyHttpServer: proxy, priv: priv, pub: pub, flysrcParser: nil} @@ -390,6 +398,20 @@ type processorsResult struct { safeSecret string } +// OpenSealed decodes and unseals a sealed token and returns its raw (unparsed) contents. +func OpenSealed(b64Secret string, pub, priv *[32]byte) ([]byte, error) { + ctSecret, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Secret)) + if err != nil { + return nil, fmt.Errorf("bad Proxy-Tokenizer encoding: %w", err) + } + + jsonSecret, ok := box.OpenAnonymous(nil, ctSecret, pub, priv) + if !ok { + return nil, errors.New("failed Proxy-Tokenizer decryption") + } + return jsonSecret, nil +} + func (t *tokenizer) processorsFromRequest(req *http.Request) (*processorsResult, error) { hdrs := req.Header[headerProxyTokenizer] result := &processorsResult{ @@ -402,14 +424,9 @@ func (t *tokenizer) processorsFromRequest(req *http.Request) (*processorsResult, return result, err } - ctSecret, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Secret)) + jsonSecret, err := OpenSealed(b64Secret, t.pub, t.priv) if err != nil { - return result, fmt.Errorf("bad Proxy-Tokenizer encoding: %w", err) - } - - jsonSecret, ok := box.OpenAnonymous(nil, ctSecret, t.pub, t.priv) - if !ok { - return result, errors.New("failed Proxy-Tokenizer decryption") + return result, err } secret := new(Secret) From d89307b4a0bb9c0d15d7b4f6f3cc2b3170644787 Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 6 Apr 2026 16:29:35 -1000 Subject: [PATCH 2/6] dont forget about the use flysrc flag! --- cmd/tokenizer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tokenizer/README.md b/cmd/tokenizer/README.md index 922145f..6bccc85 100644 --- a/cmd/tokenizer/README.md +++ b/cmd/tokenizer/README.md @@ -15,7 +15,7 @@ To run a test server locally: source ./.envrc # run server -go run . serve +go run . serve -use-flysrc=true ``` The `github.com/superfly/tokenizer/cmd/curl` package has instructions for sending requests via this server with a test client. From b3a10f0daf2a6cd095257415cb6243322a515111 Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 6 Apr 2026 20:38:26 -1000 Subject: [PATCH 3/6] Allow sealing using either SEAL_KEY or OPEN_KEY. --- cmd/tokenizer/seal.go | 22 +++++++++++++++++----- cmd/tokenizer/unseal.go | 9 ++------- tokenizer.go | 6 +++--- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/cmd/tokenizer/seal.go b/cmd/tokenizer/seal.go index 3416ff8..31992bf 100644 --- a/cmd/tokenizer/seal.go +++ b/cmd/tokenizer/seal.go @@ -10,26 +10,38 @@ import ( ) var sealUsage = ` -tokenizer seal parses a json configuration for a sealed token and seals it with the the tokenizer's OPEN_KEY +tokenizer seal parses a json configuration for a sealed token and seals it with the the +tokenizer's SEAL_KEY. It can automatically derive the sealing key from OPEN_KEY if provided. Environment variables: SEAL_KEY - Hex encoded sealing key, as generated by the "tokenizer sealkey" command. + OPEN_KEY - Hex encoded curve25519 private key. You can provide 32 + random, hex encoded bytes. The log output will contain + the associated public key. DEBUG - Display debugging output when true. ` func runSeal(cmd string, args []string) { fs := flag.NewFlagSet(cmd, flag.ContinueOnError) var ( - sealKey = fs.String("seal-key", os.Getenv("SEAL_KEY"), "public key for sealing tokens") + openKey = fs.String("open-key", os.Getenv("OPEN_KEY"), "private key for opening sealed tokens") + sealKeyP = fs.String("seal-key", os.Getenv("SEAL_KEY"), "public key for sealing tokens") jsonStr = fs.String("json", "", "JSON tokenizer secret to seal") jsonFile = fs.String("json-file", "", "file containing JSON tokenizer secret to seal") debug = fs.String("debug", os.Getenv("DEBUG"), "display debugging output when true") ) parseFlags(fs, sealUsage, args) - if *sealKey == "" { - fmt.Fprintf(os.Stderr, "missing SEAL_KEY\n") + var sealKey string + switch { + case *sealKeyP != "": + sealKey = *sealKeyP + case *openKey != "": + tkz := tokenizer.NewTokenizer(*openKey) + sealKey = tkz.SealKey() + default: + fmt.Fprintf(os.Stderr, "missing OPEN_KEY or SEAL_KEY\n") os.Exit(1) } @@ -67,7 +79,7 @@ func runSeal(cmd string, args []string) { } } - sealed, err := secret.Seal(*sealKey) + sealed, err := secret.Seal(sealKey) if err != nil { fmt.Fprintf(os.Stderr, "sealing error: %v\n", err) os.Exit(1) diff --git a/cmd/tokenizer/unseal.go b/cmd/tokenizer/unseal.go index 9dee151..70d0170 100644 --- a/cmd/tokenizer/unseal.go +++ b/cmd/tokenizer/unseal.go @@ -36,13 +36,8 @@ func runUnseal(cmd string, args []string) { os.Exit(1) } - priv, pub, err := tokenizer.ParseOpenKey(*openKey) - if err != nil { - fmt.Fprintf(os.Stderr, "OPEN_KEY: %v\n", err) - os.Exit(1) - } - - secret, err := tokenizer.OpenSealed(*token, pub, priv) + tkz := tokenizer.NewTokenizer(*openKey) + secret, err := tkz.OpenSealed(*token) if err != nil { fmt.Fprintf(os.Stderr, "token: %v\n", err) os.Exit(1) diff --git a/tokenizer.go b/tokenizer.go index c023891..f2bb5ce 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -399,13 +399,13 @@ type processorsResult struct { } // OpenSealed decodes and unseals a sealed token and returns its raw (unparsed) contents. -func OpenSealed(b64Secret string, pub, priv *[32]byte) ([]byte, error) { +func (t *tokenizer) OpenSealed(b64Secret string) ([]byte, error) { ctSecret, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Secret)) if err != nil { return nil, fmt.Errorf("bad Proxy-Tokenizer encoding: %w", err) } - jsonSecret, ok := box.OpenAnonymous(nil, ctSecret, pub, priv) + jsonSecret, ok := box.OpenAnonymous(nil, ctSecret, t.pub, t.priv) if !ok { return nil, errors.New("failed Proxy-Tokenizer decryption") } @@ -424,7 +424,7 @@ func (t *tokenizer) processorsFromRequest(req *http.Request) (*processorsResult, return result, err } - jsonSecret, err := OpenSealed(b64Secret, t.pub, t.priv) + jsonSecret, err := t.OpenSealed(b64Secret) if err != nil { return result, err } From a505114a1ba373c7f39e727f6069e4eff7b20638 Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 6 Apr 2026 20:51:13 -1000 Subject: [PATCH 4/6] minor code golf --- cmd/tokenizer/seal.go | 3 +-- cmd/tokenizer/unseal.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/tokenizer/seal.go b/cmd/tokenizer/seal.go index 31992bf..59119af 100644 --- a/cmd/tokenizer/seal.go +++ b/cmd/tokenizer/seal.go @@ -38,8 +38,7 @@ func runSeal(cmd string, args []string) { case *sealKeyP != "": sealKey = *sealKeyP case *openKey != "": - tkz := tokenizer.NewTokenizer(*openKey) - sealKey = tkz.SealKey() + sealKey = tokenizer.NewTokenizer(*openKey).SealKey() default: fmt.Fprintf(os.Stderr, "missing OPEN_KEY or SEAL_KEY\n") os.Exit(1) diff --git a/cmd/tokenizer/unseal.go b/cmd/tokenizer/unseal.go index 70d0170..1c94eeb 100644 --- a/cmd/tokenizer/unseal.go +++ b/cmd/tokenizer/unseal.go @@ -36,8 +36,7 @@ func runUnseal(cmd string, args []string) { os.Exit(1) } - tkz := tokenizer.NewTokenizer(*openKey) - secret, err := tkz.OpenSealed(*token) + secret, err := tokenizer.NewTokenizer(*openKey).OpenSealed(*token) if err != nil { fmt.Fprintf(os.Stderr, "token: %v\n", err) os.Exit(1) From 4eba2579a2c10f59d643a147933ce5b981cb639a Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 6 Apr 2026 20:56:01 -1000 Subject: [PATCH 5/6] Mention sealing and unsealing on deployed tokenizers. --- docs/UserGuide.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 479e559..a71a9a4 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -11,7 +11,7 @@ To seal a token you need the sealing key of your tokenizer. ## Command line To seal a token with the command line -* Place the sealing key in your environment as `SEAL_KEY`. +* Place the sealing key in your environment as `SEAL_KEY`. Alternately place the unsealing key in your environment as `OPEN_KEY`. * Construct the sealing information as json, perhaps putting it into a file such as `token.json`. * Run `go run ./cmd/tokenizer -json-file token.json` or `go run ./cmd/tokenizer -json $JSONOBJ`. @@ -37,6 +37,14 @@ go run ./cmd/tokenizer seal -json-file token.json go run ./cmd/tokenizer seal -json '{"inject_processor":{"token":"MY_SECRET_TOKEN"},"allowed_hosts":["timflyio-go-example.fly.dev"],"fly_src_auth":{"allowed_orgs":["tim-newsham"],"allowed_apps":["thenewsh"]}}' ``` +On a deployed tokenizer where `OPEN_KEY` is already set, you can seal and unseal keys without making any additional settings: + +```bash +TOKEN=$(tokenizer seal -json '{"inject_processor":{"token":"MY_SECRET_TOKEN"},"allowed_hosts":["timflyio-go-example.fly.dev"],"fly_src_auth":{"allowed_orgs":["tim-newsham"],"allowed_apps":["thenewsh"]}}') +echo $TOKEN +tokenizer unseal -token "$TOKEN" +``` + ## Go To seal a token with go, import `github.com/superfly/tokenizer` library, build up a `tokenizer.Secret` and call the `Seal` method with the sealing key. From 8747c22b129553ba176ca3a433393e79b2db0665 Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Tue, 7 Apr 2026 14:14:23 -1000 Subject: [PATCH 6/6] address two bugs from matt. --- cmd/tokenizer/version.go | 1 - tokenizer.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/tokenizer/version.go b/cmd/tokenizer/version.go index 9c0645c..e61388f 100644 --- a/cmd/tokenizer/version.go +++ b/cmd/tokenizer/version.go @@ -14,7 +14,6 @@ tokenizer version displays the version func runVersion(cmd string, args []string) { fs := flag.NewFlagSet(cmd, flag.ContinueOnError) parseFlags(fs, versionUsage, args) - parseFlags(fs, versionUsage, args) fmt.Fprintln(os.Stderr, versionString()) } diff --git a/tokenizer.go b/tokenizer.go index f2bb5ce..e74e8bd 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -109,7 +109,7 @@ func ParseOpenKey(openKey string) (*[32]byte, *[32]byte, error) { return nil, nil, fmt.Errorf("bad private key") } if len(privBytes) != 32 { - fmt.Errorf("bad private key size: %d", len(privBytes)) + return nil, nil, fmt.Errorf("bad private key size: %d", len(privBytes)) } priv := (*[32]byte)(privBytes) pub := new([32]byte)