From 71958c5601f8023237a369f8227d6e8085a9088f Mon Sep 17 00:00:00 2001 From: renatoathaydes Date: Wed, 11 Nov 2026 20:46:02 +0100 Subject: [PATCH] Add devpod cp to copy files to and from a workspace `devpod cp SRC WORKSPACE:DEST` and `devpod cp WORKSPACE:SRC DEST` copy files and directories between the local machine and a workspace's container, with scp's WORKSPACE:PATH syntax. A relative path in the workspace is relative to the workspace folder (read from the SSH config entry that `devpod up` writes), and where things land follows the rules of `docker cp`. It copies as the workspace's user (`--user` to change), over SFTP through the same connection `devpod ssh` makes. Each file is written to a private temporary file that is renamed over the destination, so that copying a file onto itself through the mounted workspace folder doesn't truncate it, read-only files can be replaced, and private files are never readable by others. Permissions, modification times and symlinks are kept, names that the destination can't hold safely are skipped, and files are copied concurrently. The SSH server in workspaces now serves each SFTP session from a `devpod helper sftp-server` process started in the user's home directory, like OpenSSH's sftp-server. Serving it in-process with sftp.WithServerWorkingDirectory made relative symlink targets absolute, for any SFTP client. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 + cmd/cp.go | 271 ++++++++++ cmd/cp_test.go | 51 ++ cmd/helper/helper.go | 1 + cmd/helper/sftp_server.go | 40 ++ cmd/root.go | 1 + .../connect-to-a-workspace.mdx | 28 ++ go.mod | 4 +- pkg/filetransfer/copy.go | 351 +++++++++++++ pkg/filetransfer/copy_test.go | 469 ++++++++++++++++++ pkg/filetransfer/fs.go | 250 ++++++++++ pkg/ssh/config.go | 32 ++ pkg/ssh/config_test.go | 18 + pkg/ssh/server/sftp_handler.go | 49 +- 14 files changed, 1555 insertions(+), 24 deletions(-) create mode 100644 cmd/cp.go create mode 100644 cmd/cp_test.go create mode 100644 cmd/helper/sftp_server.go create mode 100644 pkg/filetransfer/copy.go create mode 100644 pkg/filetransfer/copy_test.go create mode 100644 pkg/filetransfer/fs.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 68649a0cf..376823001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ Add new entries to the top of the "Unreleased" section. ## Unreleased +### Added + +- `devpod cp` copies files and directories to or from a workspace, such as `devpod cp ./temp/ my-workspace:/opt/` + or `devpod cp my-workspace:out.log .`. A relative path in the workspace is relative to the workspace folder. It + copies as the workspace's user (`--user` to change), over SFTP through the same connection as `devpod ssh`. + +### Fixed + +- A relative symlink created over SFTP in a workspace, such as with `sftp WORKSPACE.devpod` or an IDE's SFTP sync, + keeps its relative target. The SSH server in workspaces made it absolute (relative to the user's home + directory), because it served SFTP with a working directory, which also applied to symlink targets. It now runs + each SFTP session in a `devpod helper sftp-server` process started in the home directory, like OpenSSH's + `sftp-server`. + ## v0.10.0 (2026-11-09) ### Fixed diff --git a/cmd/cp.go b/cmd/cp.go new file mode 100644 index 000000000..50d605b28 --- /dev/null +++ b/cmd/cp.go @@ -0,0 +1,271 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "runtime" + "strings" + + "github.com/curityio/devpod/cmd/completion" + "github.com/curityio/devpod/cmd/flags" + "github.com/curityio/devpod/pkg/agent" + client2 "github.com/curityio/devpod/pkg/client" + "github.com/curityio/devpod/pkg/config" + "github.com/curityio/devpod/pkg/filetransfer" + "github.com/curityio/devpod/pkg/log" + devssh "github.com/curityio/devpod/pkg/ssh" + "github.com/curityio/devpod/pkg/tunnel" + workspace2 "github.com/curityio/devpod/pkg/workspace" + "github.com/pkg/sftp" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "golang.org/x/crypto/ssh" +) + +// CpCmd holds the cp cmd flags +type CpCmd struct { + *flags.GlobalFlags + + User string +} + +// NewCpCmd creates a new cp command +func NewCpCmd(f *flags.GlobalFlags) *cobra.Command { + cmd := &CpCmd{ + GlobalFlags: f, + } + cpCmd := &cobra.Command{ + Use: "cp [flags] SRC_PATH WORKSPACE:DEST_PATH | WORKSPACE:SRC_PATH DEST_PATH", + Short: "Copies files and directories to or from a workspace", + Long: `Copies files and directories between the local machine and a workspace's container. + +A path in the container is written as WORKSPACE:PATH, like in scp. A relative PATH is relative to the workspace +folder, and an empty one is the workspace folder itself. To copy a local file whose name has a colon in it, start +its path with ./ + +Directories are copied with everything in them. Like docker cp, a file or directory is copied into DEST_PATH if +that is an existing directory, and to DEST_PATH otherwise. A DEST_PATH that ends with / must be a directory, and is +created if it's missing. A SRC_PATH that ends with /. copies the directory's contents instead of the directory.`, + Example: ` # copy a file into the workspace folder + devpod cp my-file.txt my-workspace: + + # copy a file to a path relative to the workspace folder, under another name + devpod cp ./here/file-here.txt my-workspace:file-in-container.txt + + # copy a directory into an absolute path in the container + devpod cp ./temp/ my-workspace:/usr/local/ + + # copy a file from the container to the current directory + devpod cp my-workspace:/etc/os-release .`, + Args: cobra.ExactArgs(2), + RunE: func(cobraCmd *cobra.Command, args []string) error { + src, err := parseCpLocation(args[0], runtime.GOOS) + if err != nil { + return err + } + dst, err := parseCpLocation(args[1], runtime.GOOS) + if err != nil { + return err + } + if src.workspace != "" && dst.workspace != "" { + return fmt.Errorf("can't copy between two workspaces: only one of %s and %s can be in a workspace", args[0], args[1]) + } else if src.workspace == "" && dst.workspace == "" { + return fmt.Errorf("neither %s nor %s is in a workspace: write a path in a workspace as WORKSPACE:PATH", args[0], args[1]) + } + if src.workspace == "" { + // fail before connecting to the workspace + if _, err := os.Stat(src.path); err != nil { + return err + } + } + + devPodConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + + workspaceID := src.workspace + dst.workspace + ctx := cobraCmd.Context() + client, err := workspace2.Get(ctx, devPodConfig, []string{workspaceID}, true, true, log.Default.ErrorStreamOnly()) + if err != nil { + return err + } + + return cmd.Run(ctx, devPodConfig, client, src, dst, log.Default) + }, + ValidArgsFunction: func(rootCmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if strings.Contains(toComplete, ":") { + return nil, cobra.ShellCompDirectiveNoFileComp + } + workspaces, _ := completion.GetWorkspaceSuggestions(rootCmd, cmd.Context, cmd.Provider, nil, toComplete, log.Default) + for i := range workspaces { + workspaces[i] += ":" + } + return workspaces, cobra.ShellCompDirectiveDefault | cobra.ShellCompDirectiveNoSpace + }, + } + + cpCmd.Flags().StringVar(&cmd.User, "user", "", "The user in the container to copy as, which owns the files copied into it. Defaults to the workspace's user") + return cpCmd +} + +// cpLocation is a path given to devpod cp, in a workspace if workspace isn't empty. +type cpLocation struct { + workspace string + path string +} + +func (l cpLocation) String() string { + if l.workspace == "" { + return l.path + } + return l.workspace + ":" + l.path +} + +// parseCpLocation splits WORKSPACE:PATH, like scp: an argument is a local path if it has no colon, has a slash +// before its first colon, or (on Windows) starts with a drive letter. +func parseCpLocation(arg, goos string) (cpLocation, error) { + workspace, p, found := strings.Cut(arg, ":") + if !found || strings.ContainsAny(workspace, `/\`) || (goos == "windows" && isDriveLetter(workspace)) { + return cpLocation{path: arg}, nil + } + if workspace == "" { + return cpLocation{}, fmt.Errorf("%s has no workspace before the colon: write a path in a workspace as WORKSPACE:PATH, or a local path that starts with a colon as ./%s", arg, arg) + } + return cpLocation{workspace: workspace, path: p}, nil +} + +func isDriveLetter(s string) bool { + return len(s) == 1 && ('a' <= s[0] && s[0] <= 'z' || 'A' <= s[0] && s[0] <= 'Z') +} + +// Run copies src to dst, one of which is in the workspace client is for +func (cmd *CpCmd) Run( + ctx context.Context, + devPodConfig *config.Config, + client client2.WorkspaceClient, + src, dst cpLocation, + log log.Logger, +) error { + workspaceConfig := client.WorkspaceConfig() + user := cmd.User + if user == "" { + var err error + user, err = devssh.GetUser(workspaceConfig.ID, workspaceConfig.SSHConfigPath) + if err != nil { + return err + } + } + + workdir, err := devssh.GetWorkdir(workspaceConfig.ID, workspaceConfig.SSHConfigPath) + if err != nil { + return err + } + if workdir == "" { + workdir = path.Join("/workspaces", client.Workspace()) + } + + // lock the workspace as long as we init the connection + err = client.Lock(ctx) + if err != nil { + return err + } + defer client.Unlock() + + err = startWait(ctx, client, false, log) + if err != nil { + return err + } + + return tunnel.NewContainerTunnel(client, log.ErrorStreamOnly()). + Run(ctx, func(ctx context.Context, containerClient *ssh.Client) error { + // we have a connection to the container, make sure others can connect as well + client.Unlock() + + return withContainerSFTP(ctx, containerClient, user, workdir, log, func(sftpClient *sftp.Client) error { + var srcFS, dstFS filetransfer.FS = filetransfer.LocalFS{}, filetransfer.LocalFS{} + remote := filetransfer.SFTPFS{Client: sftpClient, WorkDir: workdir} + if src.workspace != "" { + srcFS = remote + } else { + dstFS = remote + } + + stats, err := filetransfer.Copy(srcFS, src.path, dstFS, dst.path, log) + if err != nil { + var remoteErr *filetransfer.RemoteError + if user != "root" && errors.As(err, &remoteErr) && errors.Is(remoteErr, fs.ErrPermission) { + return fmt.Errorf("copy %s to %s as user %s: %w (use --user root to copy as root)", src, dst, user, err) + } + return fmt.Errorf("copy %s to %s: %w", src, dst, err) + } + log.Donef("Copied %s from %s to %s", stats, src, dst) + return nil + }) + }, devPodConfig, nil) +} + +// withContainerSFTP starts an SSH server in the container as user, as devpod ssh does, and calls fn with an SFTP +// client connected to it. +func withContainerSFTP( + ctx context.Context, + containerClient *ssh.Client, + user, workdir string, + log log.Logger, + fn func(*sftp.Client) error, +) error { + command := fmt.Sprintf("'%s' helper ssh-server --stdio --workdir '%s'", agent.ContainerDevPodHelperLocation, workdir) + if log.GetLevel() == logrus.DebugLevel { + command += " --debug" + } + if user != "" && user != "root" { + command = fmt.Sprintf("su -c \"%s\" '%s'", command, user) + } + + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return err + } + defer stdoutReader.Close() + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return err + } + defer stdinReader.Close() + defer stdinWriter.Close() + + writer := log.ErrorStreamOnly().Writer(logrus.InfoLevel, false) + defer writer.Close() + + serverErr := make(chan error, 1) + go func() { + defer stdoutWriter.Close() + log.Debugf("Run SSH server in container: %s", command) + serverErr <- devssh.Run(ctx, containerClient, command, stdinReader, stdoutWriter, writer, nil) + }() + + sshClient, err := devssh.StdioClientWithUser(stdoutReader, stdinWriter, user, false) + if err != nil { + select { + case runErr := <-serverErr: + if runErr != nil { + return fmt.Errorf("start SSH server in container: %w", runErr) + } + default: + } + return fmt.Errorf("connect to SSH server in container: %w", err) + } + defer sshClient.Close() + + sftpClient, err := sftp.NewClient(sshClient, sftp.UseConcurrentWrites(true)) + if err != nil { + return fmt.Errorf("start SFTP session: %w", err) + } + defer sftpClient.Close() + + return fn(sftpClient) +} diff --git a/cmd/cp_test.go b/cmd/cp_test.go new file mode 100644 index 000000000..465f3a170 --- /dev/null +++ b/cmd/cp_test.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "testing" +) + +func TestParseCpLocation(t *testing.T) { + tests := []struct { + arg string + goos string + workspace string + path string + err bool + }{ + {arg: "my-file.txt", path: "my-file.txt"}, + {arg: "./here/file-here.txt", path: "./here/file-here.txt"}, + {arg: "foo:", workspace: "foo", path: ""}, + {arg: "foo:file-in-container.txt", workspace: "foo", path: "file-in-container.txt"}, + {arg: "foo:/usr/local/", workspace: "foo", path: "/usr/local/"}, + {arg: "foo:a:b", workspace: "foo", path: "a:b"}, + {arg: "./notes:2024.txt", path: "./notes:2024.txt"}, + {arg: "dir/notes:2024.txt", path: "dir/notes:2024.txt"}, + {arg: `dir\notes:2024.txt`, path: `dir\notes:2024.txt`}, + {arg: ":file", err: true}, + {arg: "c:file", goos: "linux", workspace: "c", path: "file"}, + {arg: `C:\Users\me\file.txt`, goos: "windows", path: `C:\Users\me\file.txt`}, + {arg: "C:file.txt", goos: "windows", path: "C:file.txt"}, + {arg: "foo:file.txt", goos: "windows", workspace: "foo", path: "file.txt"}, + } + for _, tc := range tests { + t.Run(tc.arg, func(t *testing.T) { + goos := tc.goos + if goos == "" { + goos = "linux" + } + location, err := parseCpLocation(tc.arg, goos) + if tc.err { + if err == nil { + t.Fatalf("expected an error, got %+v", location) + } + return + } + if err != nil { + t.Fatal(err) + } + if location.workspace != tc.workspace || location.path != tc.path { + t.Errorf("expected workspace %q and path %q, got %+v", tc.workspace, tc.path, location) + } + }) + } +} diff --git a/cmd/helper/helper.go b/cmd/helper/helper.go index 89a290732..f33d3c35c 100644 --- a/cmd/helper/helper.go +++ b/cmd/helper/helper.go @@ -24,6 +24,7 @@ func NewHelperCmd(globalFlags *flags.GlobalFlags) *cobra.Command { helperCmd.AddCommand(json.NewJSONCmd(globalFlags)) helperCmd.AddCommand(strings.NewStringsCmd(globalFlags)) helperCmd.AddCommand(NewSSHServerCmd(globalFlags)) + helperCmd.AddCommand(NewSFTPServerCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceNameCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceUIDCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceConfigCommand(globalFlags)) diff --git a/cmd/helper/sftp_server.go b/cmd/helper/sftp_server.go new file mode 100644 index 000000000..a4bc2db6b --- /dev/null +++ b/cmd/helper/sftp_server.go @@ -0,0 +1,40 @@ +package helper + +import ( + "errors" + "io" + "os" + + "github.com/curityio/devpod/cmd/flags" + "github.com/curityio/devpod/pkg/stdio" + "github.com/pkg/sftp" + "github.com/spf13/cobra" +) + +// NewSFTPServerCmd creates a command that serves SFTP over stdin and stdout, like OpenSSH's sftp-server. The SSH +// server runs it for SFTP sessions in the user's home directory, which relative paths are relative to. +func NewSFTPServerCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + return &cobra.Command{ + Use: "sftp-server", + Short: "Serves SFTP over stdin and stdout", + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, _ []string) error { + // stdout is the SFTP stream, so nothing else may be written to it + var options []sftp.ServerOption + if globalFlags.Debug { + options = append(options, sftp.WithDebug(os.Stderr)) + } + server, err := sftp.NewServer(stdio.NewStdioStream(os.Stdin, os.Stdout, false, 0), options...) + if err != nil { + return err + } + defer server.Close() + + err = server.Serve() + if errors.Is(err, io.EOF) { + return nil + } + return err + }, + } +} diff --git a/cmd/root.go b/cmd/root.go index 5d8800ebf..9662a8c43 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -119,6 +119,7 @@ func BuildRoot() *cobra.Command { rootCmd.AddCommand(NewUpCmd(globalFlags)) rootCmd.AddCommand(NewDeleteCmd(globalFlags)) rootCmd.AddCommand(NewSSHCmd(globalFlags)) + rootCmd.AddCommand(NewCpCmd(globalFlags)) rootCmd.AddCommand(NewVersionCmd()) rootCmd.AddCommand(NewStopCmd(globalFlags)) rootCmd.AddCommand(NewListCmd(globalFlags)) diff --git a/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx b/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx index 250ae9d4b..e6d134fa2 100644 --- a/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx +++ b/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx @@ -88,6 +88,34 @@ Optionally you can also define a command to run: devpod ssh my-workspace --command "echo Hello World" ``` +### Copy Files + +`devpod cp` copies files and directories between your machine and a workspace. A path in the workspace is +written as `WORKSPACE_NAME:PATH`, like with `scp`. A relative path is relative to the workspace folder, and an +empty one is the workspace folder itself: +``` +# copy a file into the workspace folder +devpod cp my-file.txt my-workspace: + +# copy a file to a path relative to the workspace folder, under another name +devpod cp ./here/file-here.txt my-workspace:file-in-container.txt + +# copy a directory into an absolute path in the container +devpod cp ./temp/ my-workspace:/opt/ + +# copy a file from the workspace to the current directory +devpod cp my-workspace:/etc/os-release . +``` + +Like `docker cp`, a file or directory is copied into the destination if that is an existing directory, and to the +destination otherwise. A destination that ends with `/` must be a directory, and is created if it's missing. A +source that ends with `/.` copies a directory's contents instead of the directory. Directories are copied with +everything in them, keeping permissions, modification times and symlinks. + +Files are copied as the workspace's user, who owns the files copied into the workspace. To write where only root +can, such as `/usr/local`, add `--user root`. To copy a local file whose name has a colon in it, start its path +with `./`. + ## IDE Commands This section shows additional commands to configure DevPod's behavior when opening a workspace. diff --git a/go.mod b/go.mod index f869b4527..9b826e36f 100644 --- a/go.mod +++ b/go.mod @@ -164,7 +164,7 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect - github.com/docker/go-units v0.5.0 // indirect + github.com/docker/go-units v0.5.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -188,7 +188,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/sync v0.12.0 golang.org/x/text v0.23.0 golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.29.0 // indirect diff --git a/pkg/filetransfer/copy.go b/pkg/filetransfer/copy.go new file mode 100644 index 000000000..c2277f5fc --- /dev/null +++ b/pkg/filetransfer/copy.go @@ -0,0 +1,351 @@ +package filetransfer + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "sync" + + "github.com/curityio/devpod/pkg/log" + "github.com/docker/go-units" + "golang.org/x/sync/errgroup" +) + +// concurrency is how many files are copied at once, which hides the latency of each SFTP request. +const concurrency = 8 + +// Stats says what Copy copied. +type Stats struct { + Files int + Bytes int64 +} + +// String summarizes stats, such as "3 files (1.2kB)". +func (s Stats) String() string { + files := "files" + if s.Files == 1 { + files = "file" + } + return fmt.Sprintf("%d %s (%s)", s.Files, files, units.HumanSize(float64(s.Bytes))) +} + +// Copy copies srcPath on src to dstPath on dst, following the rules of `docker cp`: +// +// - A file is copied into dstPath if that is a directory, and to dstPath otherwise. +// - A directory is copied into dstPath if that is a directory, and to dstPath otherwise. A srcPath that ends with +// "/." copies the directory's contents into dstPath instead. +// - A dstPath that ends with a separator must be a directory, and is created, with any missing parents, if it +// doesn't exist. +// +// Within a directory, symlinks are copied as symlinks, and files that aren't regular files, directories or symlinks +// are skipped. The permission bits and modification times are kept. Existing files are replaced: each file is +// written to a temporary file next to it and renamed over it, so that copying a file onto itself (through a mount) +// doesn't truncate it, and read-only files can be replaced. +func Copy(src FS, srcPath string, dst FS, dstPath string, log log.Logger) (Stats, error) { + contentsOnly := endsWithDot(src, srcPath) + dstMustBeDir := dstPath == "" || dst.IsSeparator(dstPath[len(dstPath)-1]) + + srcPath, err := src.Abs(srcPath) + if err != nil { + return Stats{}, err + } + dstPath, err = dst.Abs(dstPath) + if err != nil { + return Stats{}, err + } + + srcInfo, err := src.Stat(srcPath) + if err != nil { + return Stats{}, err + } + dstInfo, err := dst.Stat(dstPath) + dstExists := err == nil + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return Stats{}, err + } + + if dstExists && !dstInfo.IsDir() && (dstMustBeDir || srcInfo.IsDir()) { + return Stats{}, fmt.Errorf("%s is not a directory", dstPath) + } + target := dstPath + switch { + case dstExists && dstInfo.IsDir(): + if !srcInfo.IsDir() || !contentsOnly { + target = dst.Join(dstPath, src.Base(srcPath)) + } + case dstMustBeDir && srcInfo.IsDir(): + // dstPath becomes the copy of the directory, so only its parents are created here + err = mkdirAll(dst, dst.Dir(dstPath)) + if err != nil { + return Stats{}, err + } + case dstMustBeDir: + err = mkdirAll(dst, dstPath) + if err != nil { + return Stats{}, err + } + target = dst.Join(dstPath, src.Base(srcPath)) + } + + // List everything before writing anything, so that copying a directory into itself (which is possible when the + // container's workspace folder is a local folder mounted into it) doesn't go on forever. + c := &copier{src: src, dst: dst, log: log, created: map[string]bool{}} + entries, err := c.walk(srcPath, srcInfo, target) + if err != nil { + return Stats{}, err + } + + // Directories first, parents before children, so that files can then be copied in any order. + for _, e := range entries { + if e.info.IsDir() { + err = c.copyDir(e) + if err != nil { + return c.stats, err + } + } + } + + group, ctx := errgroup.WithContext(context.Background()) + group.SetLimit(concurrency) + for _, e := range entries { + if !e.info.IsDir() { + group.Go(func() error { + // stop at the first failure + if ctx.Err() != nil { + return nil + } + return c.copy(e) + }) + } + } + err = group.Wait() + if err != nil { + return c.stats, err + } + + // Set directories' metadata last, as writing into them changes their modification time, and a read-only + // directory can't be written into. Children go before their parents. + for i := len(entries) - 1; i >= 0; i-- { + if entries[i].info.IsDir() { + c.setDirMetadata(entries[i]) + } + } + + return c.stats, nil +} + +func mkdirAll(f FS, p string) error { + err := f.MkdirAll(p) + if err != nil { + return fmt.Errorf("create directory %s: %w", p, err) + } + return nil +} + +// endsWithDot reports whether p ends with a separator followed by ".", which means a directory's contents. +func endsWithDot(f FS, p string) bool { + return len(p) >= 2 && p[len(p)-1] == '.' && f.IsSeparator(p[len(p)-2]) +} + +type entry struct { + info fs.FileInfo + srcPath, dstPath string + // dstDir is the directory dstPath is in. + dstDir string +} + +type copier struct { + src, dst FS + log log.Logger + + // created holds the directories this copy created, which can't have anything in them yet. It's only written + // to before files are copied concurrently. + created map[string]bool + + mu sync.Mutex + stats Stats +} + +// walk lists srcPath and everything in it, skipping what can't be named on the destination. +func (c *copier) walk(srcPath string, srcInfo fs.FileInfo, target string) ([]entry, error) { + entries := []entry{{info: srcInfo, srcPath: srcPath, dstPath: target, dstDir: c.dst.Dir(target)}} + for i := 0; i < len(entries); i++ { + parent := entries[i] + if !parent.info.IsDir() { + continue + } + children, err := c.src.ReadDir(parent.srcPath) + if err != nil { + return nil, fmt.Errorf("read directory %s: %w", parent.srcPath, err) + } + for _, child := range children { + name := child.Name() + if !c.src.ValidName(name) || !c.dst.ValidName(name) { + c.log.Warnf("Skipping %q in %s, which isn't a valid file name on the destination", name, parent.srcPath) + continue + } + entries = append(entries, entry{ + info: child, + srcPath: c.src.Join(parent.srcPath, name), + dstPath: c.dst.Join(parent.dstPath, name), + dstDir: parent.dstPath, + }) + } + } + return entries, nil +} + +func (c *copier) copy(e entry) error { + mode := e.info.Mode() + switch { + case mode&fs.ModeSymlink != 0: + return c.copySymlink(e) + case mode.IsRegular(): + return c.copyFile(e) + default: + c.log.Warnf("Skipping %s, which is not a regular file, directory or symlink", e.srcPath) + return nil + } +} + +func (c *copier) copyDir(e entry) error { + if !c.created[e.dstDir] { + existing, err := c.dst.Lstat(e.dstPath) + switch { + case err == nil && !existing.IsDir(): + return fmt.Errorf("%s is not a directory", e.dstPath) + case err == nil: + // a read-only directory, maybe from an earlier copy, has to be writable until its metadata is set + if existing.Mode().Perm()&0o200 == 0 { + _ = c.dst.Chmod(e.dstPath, existing.Mode().Perm()|0o700) + } + return nil + case !errors.Is(err, fs.ErrNotExist): + return err + } + } + + c.log.Debugf("Creating directory %s", e.dstPath) + err := c.dst.Mkdir(e.dstPath) + if err != nil { + return fmt.Errorf("create directory %s: %w", e.dstPath, err) + } + c.created[e.dstPath] = true + return nil +} + +func (c *copier) copySymlink(e entry) error { + target, err := c.src.Readlink(e.srcPath) + if err != nil { + return fmt.Errorf("read symlink %s: %w", e.srcPath, err) + } + target = c.dst.FromSlash(c.src.ToSlash(target)) + + if !c.created[e.dstDir] { + err = c.removeExisting(e.dstPath) + if err != nil { + return err + } + } + c.log.Debugf("Creating symlink %s -> %s", e.dstPath, target) + err = c.dst.Symlink(target, e.dstPath) + if errors.Is(err, ErrSymlinkNotPermitted) { + c.log.Warnf("Skipping symlink %s -> %s: %v", e.srcPath, target, err) + return nil + } else if err != nil { + return fmt.Errorf("create symlink %s: %w", e.dstPath, err) + } + c.count(0) + return nil +} + +func (c *copier) copyFile(e entry) (err error) { + c.log.Debugf("Copying %s to %s", e.srcPath, e.dstPath) + in, err := c.src.Open(e.srcPath) + if err != nil { + return err + } + defer in.Close() + + tmp := c.dst.Join(e.dstDir, "."+c.dst.Base(e.dstPath)+".devpod-cp-"+randomSuffix()) + out, err := c.dst.CreateNew(tmp) + if err != nil { + return fmt.Errorf("create %s: %w", tmp, err) + } + defer func() { + if err != nil { + _ = c.dst.Remove(tmp) + } + }() + + n, err := io.Copy(out, in) + closeErr := out.Close() + if err != nil { + return fmt.Errorf("copy %s to %s: %w", e.srcPath, e.dstPath, err) + } else if closeErr != nil { + return fmt.Errorf("write %s: %w", tmp, closeErr) + } + + err = c.dst.Chmod(tmp, e.info.Mode().Perm()) + if err != nil { + return fmt.Errorf("set permissions of %s: %w", tmp, err) + } + if chtimesErr := c.dst.Chtimes(tmp, e.info.ModTime()); chtimesErr != nil { + c.log.Warnf("Set modification time of %s: %v", e.dstPath, chtimesErr) + } + + err = c.dst.Rename(tmp, e.dstPath) + if err != nil { + if existing, statErr := c.dst.Lstat(e.dstPath); statErr == nil && existing.IsDir() { + return fmt.Errorf("%s is a directory", e.dstPath) + } + return fmt.Errorf("replace %s: %w", e.dstPath, err) + } + + c.count(n) + return nil +} + +func (c *copier) count(bytes int64) { + c.mu.Lock() + defer c.mu.Unlock() + c.stats.Files++ + c.stats.Bytes += bytes +} + +func (c *copier) removeExisting(p string) error { + existing, err := c.dst.Lstat(p) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if existing.IsDir() { + return fmt.Errorf("%s is a directory", p) + } + return c.dst.Remove(p) +} + +// setDirMetadata copies a directory's permission bits and modification time. It only warns of failures: an existing +// directory that belongs to another user can be written to, but not changed in these ways. +func (c *copier) setDirMetadata(e entry) { + err := c.dst.Chmod(e.dstPath, e.info.Mode().Perm()) + if err != nil { + c.log.Warnf("Set permissions of %s: %v", e.dstPath, err) + } + err = c.dst.Chtimes(e.dstPath, e.info.ModTime()) + if err != nil { + c.log.Warnf("Set modification time of %s: %v", e.dstPath, err) + } +} + +func randomSuffix() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/pkg/filetransfer/copy_test.go b/pkg/filetransfer/copy_test.go new file mode 100644 index 000000000..9aba939b7 --- /dev/null +++ b/pkg/filetransfer/copy_test.go @@ -0,0 +1,469 @@ +package filetransfer + +import ( + "errors" + "io/fs" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/curityio/devpod/pkg/log" + "github.com/google/go-cmp/cmp" + "github.com/pkg/sftp" +) + +// tree describes a directory's contents: a file's content, "dir/" for a directory, or "-> target" for a symlink. +type tree map[string]string + +func writeTree(t *testing.T, root string, files tree) { + t.Helper() + for name, content := range files { + p := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + var err error + switch { + case content == "dir/": + err = os.MkdirAll(p, 0o755) + case len(content) > 3 && content[:3] == "-> ": + err = os.Symlink(content[3:], p) + default: + err = os.WriteFile(p, []byte(content), 0o644) + } + if err != nil { + t.Fatal(err) + } + } +} + +func readTree(t *testing.T, root string) tree { + t.Helper() + files := tree{} + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil || p == root { + return err + } + rel, _ := filepath.Rel(root, p) + rel = filepath.ToSlash(rel) + switch { + case d.Type()&fs.ModeSymlink != 0: + target, err := os.Readlink(p) + files[rel] = "-> " + target + return err + case d.IsDir(): + files[rel] = "dir/" + default: + content, err := os.ReadFile(p) + files[rel] = string(content) + return err + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return files +} + +// newSFTPFS serves the local file system over SFTP in-process, the way the container's SSH server does. +func newSFTPFS(t *testing.T, workDir string) SFTPFS { + t.Helper() + serverConn, clientConn := net.Pipe() + server, err := sftp.NewServer(serverConn) + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve() }() + client, err := sftp.NewClientPipe(clientConn, clientConn, sftp.UseConcurrentWrites(true)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = client.Close() + _ = server.Close() + }) + return SFTPFS{Client: client, WorkDir: workDir} +} + +type copyCase struct { + name string + src tree + dst tree + srcArg string // relative to the source root, with / separators + dstArg string // relative to the destination root, with / separators + expected tree // the destination root afterwards + err bool + symlinks bool +} + +var copyCases = []copyCase{ + { + name: "file into existing directory", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "", + expected: tree{"a.txt": "a"}, + }, + { + name: "file to new name", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "b.txt", + expected: tree{"b.txt": "a"}, + }, + { + name: "file into missing directory written with trailing slash", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "new/dir/", + expected: tree{"new": "dir/", "new/dir": "dir/", "new/dir/a.txt": "a"}, + }, + { + name: "file overwrites existing file", + src: tree{"a.txt": "new"}, + dst: tree{"a.txt": "old content"}, + srcArg: "a.txt", + dstArg: "a.txt", + expected: tree{"a.txt": "new"}, + }, + { + name: "file onto path with trailing slash that is a file", + src: tree{"a.txt": "a"}, + dst: tree{"b.txt": "b"}, + srcArg: "a.txt", + dstArg: "b.txt/", + err: true, + }, + { + name: "directory into existing directory", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y", "temp/empty": "dir/"}, + dst: tree{"existing.txt": "e"}, + srcArg: "temp/", + dstArg: "", + expected: tree{"existing.txt": "e", "temp": "dir/", "temp/x.txt": "x", "temp/sub": "dir/", "temp/sub/y.txt": "y", "temp/empty": "dir/"}, + }, + { + name: "directory to missing nested directory written with trailing slash", + src: tree{"temp/x.txt": "x"}, + srcArg: "temp", + dstArg: "a/b/", + expected: tree{"a": "dir/", "a/b": "dir/", "a/b/x.txt": "x"}, + }, + { + name: "directory to new path", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y"}, + srcArg: "temp", + dstArg: "copy", + expected: tree{"copy": "dir/", "copy/x.txt": "x", "copy/sub": "dir/", "copy/sub/y.txt": "y"}, + }, + { + name: "directory contents merged into existing directory", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y"}, + dst: tree{"sub/z.txt": "z", "x.txt": "old"}, + srcArg: "temp/.", + dstArg: "", + expected: tree{"x.txt": "x", "sub": "dir/", "sub/y.txt": "y", "sub/z.txt": "z"}, + }, + { + name: "directory onto file", + src: tree{"temp/x.txt": "x"}, + dst: tree{"file": "f"}, + srcArg: "temp", + dstArg: "file", + err: true, + }, + { + name: "symlinks are copied as symlinks", + src: tree{"temp/x.txt": "x", "temp/link": "-> x.txt", "temp/dangling": "-> nowhere"}, + dst: tree{"temp/link": "replaced"}, + srcArg: "temp", + dstArg: "", + expected: tree{"temp": "dir/", "temp/x.txt": "x", "temp/link": "-> x.txt", "temp/dangling": "-> nowhere"}, + symlinks: true, + }, +} + +func runCopyCases(t *testing.T, newDst func(t *testing.T, root string) FS, toDstArg func(root, rel string) string) { + for _, tc := range copyCases { + t.Run(tc.name, func(t *testing.T) { + if tc.symlinks && runtime.GOOS == "windows" { + t.Skip("creating symlinks needs privileges on Windows") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tc.src) + writeTree(t, dstRoot, tc.dst) + + srcArg := filepath.Join(srcRoot, filepath.FromSlash(tc.srcArg)) + if len(tc.srcArg) > 0 && tc.srcArg[len(tc.srcArg)-1] == '/' { + srcArg += string(os.PathSeparator) + } else if filepath.Base(tc.srcArg) == "." { + srcArg = filepath.Join(srcRoot, filepath.FromSlash(filepath.Dir(tc.srcArg))) + string(os.PathSeparator) + "." + } + + _, err := Copy(LocalFS{}, srcArg, newDst(t, dstRoot), toDstArg(dstRoot, tc.dstArg), log.Discard) + if tc.err { + if err == nil { + t.Fatal("expected an error") + } + return + } + if err != nil { + t.Fatal(err) + } + + expected := tree{} + for name, content := range tc.dst { + expected[name] = content + } + for name, content := range tc.expected { + expected[name] = content + } + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + }) + } +} + +func TestCopyLocal(t *testing.T) { + runCopyCases(t, + func(t *testing.T, root string) FS { return LocalFS{} }, + func(root, rel string) string { + p := filepath.Join(root, filepath.FromSlash(rel)) + if rel == "" || rel[len(rel)-1] == '/' { + p += string(os.PathSeparator) + } + return p + }) +} + +func TestCopyToSFTP(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + // relative to the work dir, as in a workspace + runCopyCases(t, + func(t *testing.T, root string) FS { return newSFTPFS(t, root) }, + func(root, rel string) string { return rel }) +} + +func TestCopyFromSFTP(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + remoteRoot, localRoot := t.TempDir(), t.TempDir() + writeTree(t, remoteRoot, tree{"temp/x.txt": "x", "temp/sub/y.txt": "y", "temp/link": "-> x.txt"}) + + stats, err := Copy(newSFTPFS(t, remoteRoot), "temp", LocalFS{}, localRoot, log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"temp": "dir/", "temp/x.txt": "x", "temp/sub": "dir/", "temp/sub/y.txt": "y", "temp/link": "-> x.txt"} + if diff := cmp.Diff(expected, readTree(t, localRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + if stats.Files != 3 || stats.Bytes != 2 { + t.Errorf("expected 3 files of 2 bytes, got %+v", stats) + } +} + +func TestCopyKeepsModeAndModTime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows has no executable bit") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + script := filepath.Join(srcRoot, "dir", "script.sh") + writeTree(t, srcRoot, tree{"dir/script.sh": "#!/bin/sh"}) + if err := os.Chmod(script, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(srcRoot, "dir"), 0o700); err != nil { + t.Fatal(err) + } + mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + for _, p := range []string{script, filepath.Join(srcRoot, "dir")} { + if err := os.Chtimes(p, mtime, mtime); err != nil { + t.Fatal(err) + } + } + + _, err := Copy(LocalFS{}, filepath.Join(srcRoot, "dir"), newSFTPFS(t, dstRoot), "", log.Discard) + if err != nil { + t.Fatal(err) + } + + for p, mode := range map[string]fs.FileMode{"dir/script.sh": 0o750, "dir": 0o700 | fs.ModeDir} { + info, err := os.Stat(filepath.Join(dstRoot, p)) + if err != nil { + t.Fatal(err) + } + if info.Mode() != mode { + t.Errorf("%s: expected mode %v, got %v", p, mode, info.Mode()) + } + if !info.ModTime().Equal(mtime) { + t.Errorf("%s: expected modification time %v, got %v", p, mtime, info.ModTime()) + } + } +} + +func TestCopyDirectoryIntoItself(t *testing.T) { + root := t.TempDir() + writeTree(t, root, tree{"dir/x.txt": "x"}) + + _, err := Copy(LocalFS{}, filepath.Join(root, "dir"), LocalFS{}, filepath.Join(root, "dir", "inner")+string(os.PathSeparator), log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"dir": "dir/", "dir/x.txt": "x", "dir/inner": "dir/", "dir/inner/x.txt": "x"} + if diff := cmp.Diff(expected, readTree(t, root)); diff != "" { + t.Errorf("unexpected tree (-want +got):\n%s", diff) + } +} + +func TestCopyFileOntoItself(t *testing.T) { + // as when the workspace folder is the local folder mounted into the container + root := t.TempDir() + writeTree(t, root, tree{"a.txt": "content", "dir/b.txt": "b"}) + + // devpod cp a.txt foo: and devpod cp dir foo:, from the workspace folder + for _, p := range []string{"a.txt", "dir"} { + _, err := Copy(LocalFS{}, filepath.Join(root, p), LocalFS{}, root+string(os.PathSeparator), log.Discard) + if err != nil { + t.Fatal(err) + } + } + + expected := tree{"a.txt": "content", "dir": "dir/", "dir/b.txt": "b"} + if diff := cmp.Diff(expected, readTree(t, root)); diff != "" { + t.Errorf("unexpected tree (-want +got):\n%s", diff) + } +} + +func TestCopyReadOnlyTreeTwice(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows has no read-only directories") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tree{"repo/objects/ab/cdef": "object"}) + for p, mode := range map[string]fs.FileMode{"repo/objects/ab/cdef": 0o444, "repo/objects/ab": 0o555} { + if err := os.Chmod(filepath.Join(srcRoot, p), mode); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + // let t.TempDir remove them + for _, root := range []string{srcRoot, dstRoot} { + _ = os.Chmod(filepath.Join(root, "repo/objects/ab"), 0o755) + } + }) + + for i := 0; i < 2; i++ { + _, err := Copy(LocalFS{}, filepath.Join(srcRoot, "repo"), newSFTPFS(t, dstRoot), "", log.Discard) + if err != nil { + t.Fatalf("copy %d: %v", i+1, err) + } + } + + expected := tree{"repo": "dir/", "repo/objects": "dir/", "repo/objects/ab": "dir/", "repo/objects/ab/cdef": "object"} + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + info, err := os.Stat(filepath.Join(dstRoot, "repo/objects/ab")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o555 { + t.Errorf("expected the directory to be read-only again, got %v", info.Mode()) + } +} + +// strictFS rejects names that start with "bad", as Windows rejects names with a backslash or colon. +type strictFS struct{ LocalFS } + +func (f strictFS) ValidName(name string) bool { + return f.LocalFS.ValidName(name) && !strings.HasPrefix(name, "bad") +} + +func TestCopySkipsInvalidNames(t *testing.T) { + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tree{"temp/ok.txt": "ok", "temp/bad.txt": "bad", "temp/bad-dir/x.txt": "x"}) + + stats, err := Copy(LocalFS{}, filepath.Join(srcRoot, "temp"), strictFS{}, dstRoot, log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"temp": "dir/", "temp/ok.txt": "ok"} + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + if stats.Files != 1 { + t.Errorf("expected 1 file, got %+v", stats) + } +} + +func TestValidName(t *testing.T) { + for _, name := range []string{"", ".", "..", "a/b"} { + if (SFTPFS{}).ValidName(name) || (LocalFS{}).ValidName(name) { + t.Errorf("expected %q to be invalid", name) + } + } + for _, name := range []string{"a.txt", ".hidden", "..."} { + if !(SFTPFS{}).ValidName(name) || !(LocalFS{}).ValidName(name) { + t.Errorf("expected %q to be valid", name) + } + } + for _, name := range []string{`..\..\evil`, "a:b"} { + if !(SFTPFS{}).ValidName(name) { + t.Errorf("expected %q to be valid in a container", name) + } + if (LocalFS{}).ValidName(name) == (runtime.GOOS == "windows") { + t.Errorf("expected %q to be valid locally only if not on Windows", name) + } + } +} + +func TestSFTPCreateNewIsPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + root := t.TempDir() + remote := newSFTPFS(t, root) + f, err := remote.CreateNew(filepath.Join(root, "secret")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + info, err := os.Stat(filepath.Join(root, "secret")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("expected a new file to be 0600 until it's written, got %v", info.Mode()) + } +} + +func TestSFTPErrorsAreRemote(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + root := t.TempDir() + _, err := Copy(newSFTPFS(t, root), "missing.txt", LocalFS{}, t.TempDir(), log.Discard) + var remoteErr *RemoteError + if !errors.As(err, &remoteErr) || !errors.Is(err, fs.ErrNotExist) { + t.Errorf("expected a remote not-exist error, got %v", err) + } + + _, err = Copy(LocalFS{}, filepath.Join(root, "missing.txt"), newSFTPFS(t, root), "", log.Discard) + if errors.As(err, &remoteErr) || !errors.Is(err, fs.ErrNotExist) { + t.Errorf("expected a local not-exist error, got %v", err) + } +} diff --git a/pkg/filetransfer/fs.go b/pkg/filetransfer/fs.go new file mode 100644 index 000000000..f91b85d3c --- /dev/null +++ b/pkg/filetransfer/fs.go @@ -0,0 +1,250 @@ +// Package filetransfer copies files and directories between two file systems, such as the local one and a dev +// container's, reached over SFTP. +package filetransfer + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/pkg/sftp" +) + +// ErrSymlinkNotPermitted is returned by Symlink on Windows when the user may not create symlinks, which needs +// Developer Mode or an administrator. +var ErrSymlinkNotPermitted = errors.New("creating symlinks isn't permitted") + +// FS is a file system that files can be copied from or to. +type FS interface { + // Abs makes p absolute. + Abs(p string) (string, error) + Join(elem ...string) string + Base(p string) string + Dir(p string) string + IsSeparator(c byte) bool + // ValidName reports whether name can be used as a single file name, so that it can't refer to another + // directory. + ValidName(name string) bool + // ToSlash and FromSlash convert a symlink target between this file system's separator and /. + ToSlash(p string) string + FromSlash(p string) string + + Stat(p string) (fs.FileInfo, error) + Lstat(p string) (fs.FileInfo, error) + // ReadDir lists a directory without following symlinks. + ReadDir(p string) ([]fs.FileInfo, error) + Readlink(p string) (string, error) + + Open(p string) (io.ReadCloser, error) + // CreateNew creates a file that doesn't exist yet, which only its owner can read and write. + CreateNew(p string) (io.WriteCloser, error) + // Rename moves oldPath to newPath, replacing newPath if it exists and isn't a directory. + Rename(oldPath, newPath string) error + Mkdir(p string) error + MkdirAll(p string) error + Remove(p string) error + Symlink(target, p string) error + Chmod(p string, mode fs.FileMode) error + Chtimes(p string, mtime time.Time) error +} + +// LocalFS is the file system of the machine DevPod runs on. +type LocalFS struct{} + +var _ FS = LocalFS{} + +func (LocalFS) Abs(p string) (string, error) { return filepath.Abs(p) } +func (LocalFS) Join(elem ...string) string { return filepath.Join(elem...) } +func (LocalFS) Base(p string) string { return filepath.Base(p) } +func (LocalFS) Dir(p string) string { return filepath.Dir(p) } +func (LocalFS) IsSeparator(c byte) bool { return os.IsPathSeparator(c) } +func (LocalFS) ToSlash(p string) string { return filepath.ToSlash(p) } +func (LocalFS) FromSlash(p string) string { return filepath.FromSlash(p) } +func (LocalFS) Readlink(p string) (string, error) { return os.Readlink(p) } +func (LocalFS) Open(p string) (io.ReadCloser, error) { return os.Open(p) } +func (LocalFS) Mkdir(p string) error { return os.Mkdir(p, 0o755) } +func (LocalFS) MkdirAll(p string) error { return os.MkdirAll(p, 0o755) } +func (LocalFS) Remove(p string) error { return os.Remove(p) } +func (LocalFS) Chmod(p string, mode fs.FileMode) error { return os.Chmod(p, mode) } +func (LocalFS) Chtimes(p string, mtime time.Time) error { return os.Chtimes(p, mtime, mtime) } + +func (LocalFS) ValidName(name string) bool { + invalid := `/` + if runtime.GOOS == "windows" { + // a colon names an alternate data stream + invalid = `/\:` + } + return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, invalid) +} + +func (LocalFS) CreateNew(p string) (io.WriteCloser, error) { + return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) +} + +func (LocalFS) Rename(oldPath, newPath string) error { + err := os.Rename(oldPath, newPath) + if err != nil && runtime.GOOS == "windows" { + // Windows can't replace a read-only file + if info, statErr := os.Lstat(newPath); statErr == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o200 == 0 { + if os.Chmod(newPath, 0o600) == nil { + return os.Rename(oldPath, newPath) + } + } + } + return err +} + +func (LocalFS) Symlink(target, p string) error { + err := os.Symlink(target, p) + // ERROR_PRIVILEGE_NOT_HELD + if err != nil && runtime.GOOS == "windows" && errors.Is(err, syscall.Errno(1314)) { + return errors.Join(ErrSymlinkNotPermitted, err) + } + return err +} + +func (LocalFS) Stat(p string) (fs.FileInfo, error) { + info, err := os.Stat(p) + return localInfo(info), err +} + +func (LocalFS) Lstat(p string) (fs.FileInfo, error) { + info, err := os.Lstat(p) + return localInfo(info), err +} + +func (LocalFS) ReadDir(p string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(p) + if err != nil { + return nil, err + } + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return nil, err + } + infos = append(infos, localInfo(info)) + } + return infos, nil +} + +// localInfo drops the group and world write bits on Windows, which reports every writable file as 0666 and every +// directory as 0777, so that files copied into a container aren't writable by everyone. +func localInfo(info fs.FileInfo) fs.FileInfo { + if info == nil || runtime.GOOS != "windows" { + return info + } + return windowsInfo{info} +} + +type windowsInfo struct{ fs.FileInfo } + +func (i windowsInfo) Mode() fs.FileMode { return i.FileInfo.Mode() &^ 0o022 } + +// RemoteError is an error from an SFTPFS, so that it can be told apart from a local one. +type RemoteError struct { + Err error +} + +func (e *RemoteError) Error() string { return e.Err.Error() } +func (e *RemoteError) Unwrap() error { return e.Err } + +func remote(err error) error { + if err == nil { + return nil + } + return &RemoteError{Err: err} +} + +// SFTPFS is a file system reached over SFTP, such as a dev container's. +type SFTPFS struct { + Client *sftp.Client + // WorkDir is what relative paths are relative to. + WorkDir string +} + +var _ FS = SFTPFS{} + +func (s SFTPFS) Abs(p string) (string, error) { + if path.IsAbs(p) { + return path.Clean(p), nil + } + return path.Join(s.WorkDir, p), nil +} + +func (SFTPFS) Join(elem ...string) string { return path.Join(elem...) } +func (SFTPFS) Base(p string) string { return path.Base(p) } +func (SFTPFS) Dir(p string) string { return path.Dir(p) } +func (SFTPFS) IsSeparator(c byte) bool { return c == '/' } +func (SFTPFS) ToSlash(p string) string { return p } +func (SFTPFS) FromSlash(p string) string { return p } + +func (SFTPFS) ValidName(name string) bool { + return name != "" && name != "." && name != ".." && !strings.Contains(name, "/") +} + +func (s SFTPFS) Stat(p string) (fs.FileInfo, error) { + info, err := s.Client.Stat(p) + return info, remote(err) +} + +func (s SFTPFS) Lstat(p string) (fs.FileInfo, error) { + info, err := s.Client.Lstat(p) + return info, remote(err) +} + +func (s SFTPFS) ReadDir(p string) ([]fs.FileInfo, error) { + infos, err := s.Client.ReadDir(p) + return infos, remote(err) +} + +func (s SFTPFS) Readlink(p string) (string, error) { + target, err := s.Client.ReadLink(p) + return target, remote(err) +} + +func (s SFTPFS) Open(p string) (io.ReadCloser, error) { + f, err := s.Client.Open(p) + if err != nil { + return nil, remote(err) + } + return f, nil +} + +func (s SFTPFS) CreateNew(p string) (io.WriteCloser, error) { + f, err := s.Client.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if err != nil { + return nil, remote(err) + } + // the server creates files as 0644, so restrict it before writing anything + err = f.Chmod(0o600) + if err != nil { + _ = f.Close() + _ = s.Client.Remove(p) + return nil, remote(err) + } + return f, nil +} + +func (s SFTPFS) Rename(oldPath, newPath string) error { + return remote(s.Client.PosixRename(oldPath, newPath)) +} + +func (s SFTPFS) Mkdir(p string) error { return remote(s.Client.Mkdir(p)) } +func (s SFTPFS) MkdirAll(p string) error { return remote(s.Client.MkdirAll(p)) } +func (s SFTPFS) Remove(p string) error { return remote(s.Client.Remove(p)) } +func (s SFTPFS) Symlink(target, p string) error { return remote(s.Client.Symlink(target, p)) } +func (s SFTPFS) Chmod(p string, m fs.FileMode) error { + return remote(s.Client.Chmod(p, m)) +} +func (s SFTPFS) Chtimes(p string, mtime time.Time) error { + return remote(s.Client.Chtimes(p, mtime, mtime)) +} diff --git a/pkg/ssh/config.go b/pkg/ssh/config.go index ead3f8c9f..c2dbe3445 100644 --- a/pkg/ssh/config.go +++ b/pkg/ssh/config.go @@ -163,6 +163,38 @@ func GetUser(workspaceID string, sshConfigPath string) (string, error) { return user, nil } +// GetWorkdir returns the folder DevPod's SSH config entry for the workspace opens, which `devpod up` sets to the +// workspace folder in the container. It returns "" if the entry doesn't set one. +func GetWorkdir(workspaceID string, sshConfigPath string) (string, error) { + path, err := ResolveSSHConfigPath(sshConfigPath) + if err != nil { + return "", errors.Wrap(err, "Invalid ssh config path") + } + + workdir := "" + _, err = transformHostSection(path, workspaceID+"."+"devpod", func(line string) string { + if strings.HasPrefix(strings.TrimSpace(line), "ProxyCommand ") { + workdir = parseWorkdirFlag(line) + } + return line + }) + if err != nil { + return "", err + } + + return workdir, nil +} + +// parseWorkdirFlag returns the value of the `--workdir ""` that addHostSection adds to the ProxyCommand. +func parseWorkdirFlag(proxyCommand string) string { + _, value, found := strings.Cut(proxyCommand, ` --workdir "`) + if !found { + return "" + } + value, _, _ = strings.Cut(value, `"`) + return value +} + func RemoveFromConfig(workspaceID string, sshConfigPath string, log log.Logger) error { configLock.Lock() defer configLock.Unlock() diff --git a/pkg/ssh/config_test.go b/pkg/ssh/config_test.go index ced975b9d..d4b32feb8 100644 --- a/pkg/ssh/config_test.go +++ b/pkg/ssh/config_test.go @@ -322,3 +322,21 @@ Host testhost }) } } + +func TestParseWorkdirFlag(t *testing.T) { + for _, workdir := range []string{"", "/workspaces/foo", "/home/me/my project"} { + config, err := addHostSection("", "/path/to/devpod", "foo.devpod", "me", "default", "foo", workdir, "", false, true, "") + if err != nil { + t.Fatal(err) + } + got := "" + for _, line := range strings.Split(config, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "ProxyCommand ") { + got = parseWorkdirFlag(line) + } + } + if got != workdir { + t.Errorf("expected workdir %q, got %q from:\n%s", workdir, got, config) + } + } +} diff --git a/pkg/ssh/server/sftp_handler.go b/pkg/ssh/server/sftp_handler.go index 514552832..4a32bc326 100644 --- a/pkg/ssh/server/sftp_handler.go +++ b/pkg/ssh/server/sftp_handler.go @@ -1,49 +1,54 @@ package server import ( - "errors" - "io" + "os" + "os/exec" "github.com/curityio/devpod/pkg/command" "github.com/curityio/devpod/pkg/log" "github.com/curityio/devpod/pkg/ssh/sshd" - "github.com/pkg/sftp" "github.com/sirupsen/logrus" ) +// sftpHandler serves an SFTP session from a `helper sftp-server` process started in the user's home directory, the +// way OpenSSH runs sftp-server, so that relative paths are relative to the home directory. It doesn't serve SFTP +// in-process with sftp.WithServerWorkingDirectory, because that also makes relative symlink targets absolute. func sftpHandler(sess sshd.Session, currentUser string, log log.Logger) { - writer := log.Writer(logrus.DebugLevel, false) - defer writer.Close() - user := sess.User() if user == currentUser { user = "" } - workingDir, _ := command.GetHome(user) - serverOptions := []sftp.ServerOption{ - sftp.WithDebug(writer), - sftp.WithServerWorkingDirectory(workingDir), + workingDir, err := command.GetHome(user) + if err != nil { + log.Debugf("sftp server: find home directory of %q: %v", user, err) } - server, err := sftp.NewServer( - sess, - serverOptions..., - ) + + executable, err := os.Executable() if err != nil { - log.Debugf("sftp server init error: %s\n", err) + log.Debugf("sftp server: find executable: %v", err) + _ = sess.Exit(1) return } - defer server.Close() - // serve - err = server.Serve() - if errors.Is(err, io.EOF) { - _ = sess.Exit(0) - return + args := []string{"helper", "sftp-server"} + if log.GetLevel() == logrus.DebugLevel { + args = append(args, "--debug") } + writer := log.Writer(logrus.DebugLevel, false) + defer writer.Close() + + cmd := exec.Command(executable, args...) + cmd.Dir = workingDir + cmd.Stdin = sess + cmd.Stdout = sess + cmd.Stderr = writer + err = cmd.Run() if err != nil { log.Debugf("sftp server completed with error: %v", err) + _ = sess.Exit(1) + return } - _ = sess.Exit(1) + _ = sess.Exit(0) }