diff --git a/CHANGELOG.md b/CHANGELOG.md index 68649a0cf..376823001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ Add new entries to the top of the "Unreleased" section. ## Unreleased +### Added + +- `devpod cp` copies files and directories to or from a workspace, such as `devpod cp ./temp/ my-workspace:/opt/` + or `devpod cp my-workspace:out.log .`. A relative path in the workspace is relative to the workspace folder. It + copies as the workspace's user (`--user` to change), over SFTP through the same connection as `devpod ssh`. + +### Fixed + +- A relative symlink created over SFTP in a workspace, such as with `sftp WORKSPACE.devpod` or an IDE's SFTP sync, + keeps its relative target. The SSH server in workspaces made it absolute (relative to the user's home + directory), because it served SFTP with a working directory, which also applied to symlink targets. It now runs + each SFTP session in a `devpod helper sftp-server` process started in the home directory, like OpenSSH's + `sftp-server`. + ## v0.10.0 (2026-11-09) ### Fixed diff --git a/cmd/cp.go b/cmd/cp.go new file mode 100644 index 000000000..50d605b28 --- /dev/null +++ b/cmd/cp.go @@ -0,0 +1,271 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "runtime" + "strings" + + "github.com/curityio/devpod/cmd/completion" + "github.com/curityio/devpod/cmd/flags" + "github.com/curityio/devpod/pkg/agent" + client2 "github.com/curityio/devpod/pkg/client" + "github.com/curityio/devpod/pkg/config" + "github.com/curityio/devpod/pkg/filetransfer" + "github.com/curityio/devpod/pkg/log" + devssh "github.com/curityio/devpod/pkg/ssh" + "github.com/curityio/devpod/pkg/tunnel" + workspace2 "github.com/curityio/devpod/pkg/workspace" + "github.com/pkg/sftp" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "golang.org/x/crypto/ssh" +) + +// CpCmd holds the cp cmd flags +type CpCmd struct { + *flags.GlobalFlags + + User string +} + +// NewCpCmd creates a new cp command +func NewCpCmd(f *flags.GlobalFlags) *cobra.Command { + cmd := &CpCmd{ + GlobalFlags: f, + } + cpCmd := &cobra.Command{ + Use: "cp [flags] SRC_PATH WORKSPACE:DEST_PATH | WORKSPACE:SRC_PATH DEST_PATH", + Short: "Copies files and directories to or from a workspace", + Long: `Copies files and directories between the local machine and a workspace's container. + +A path in the container is written as WORKSPACE:PATH, like in scp. A relative PATH is relative to the workspace +folder, and an empty one is the workspace folder itself. To copy a local file whose name has a colon in it, start +its path with ./ + +Directories are copied with everything in them. Like docker cp, a file or directory is copied into DEST_PATH if +that is an existing directory, and to DEST_PATH otherwise. A DEST_PATH that ends with / must be a directory, and is +created if it's missing. A SRC_PATH that ends with /. copies the directory's contents instead of the directory.`, + Example: ` # copy a file into the workspace folder + devpod cp my-file.txt my-workspace: + + # copy a file to a path relative to the workspace folder, under another name + devpod cp ./here/file-here.txt my-workspace:file-in-container.txt + + # copy a directory into an absolute path in the container + devpod cp ./temp/ my-workspace:/usr/local/ + + # copy a file from the container to the current directory + devpod cp my-workspace:/etc/os-release .`, + Args: cobra.ExactArgs(2), + RunE: func(cobraCmd *cobra.Command, args []string) error { + src, err := parseCpLocation(args[0], runtime.GOOS) + if err != nil { + return err + } + dst, err := parseCpLocation(args[1], runtime.GOOS) + if err != nil { + return err + } + if src.workspace != "" && dst.workspace != "" { + return fmt.Errorf("can't copy between two workspaces: only one of %s and %s can be in a workspace", args[0], args[1]) + } else if src.workspace == "" && dst.workspace == "" { + return fmt.Errorf("neither %s nor %s is in a workspace: write a path in a workspace as WORKSPACE:PATH", args[0], args[1]) + } + if src.workspace == "" { + // fail before connecting to the workspace + if _, err := os.Stat(src.path); err != nil { + return err + } + } + + devPodConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + + workspaceID := src.workspace + dst.workspace + ctx := cobraCmd.Context() + client, err := workspace2.Get(ctx, devPodConfig, []string{workspaceID}, true, true, log.Default.ErrorStreamOnly()) + if err != nil { + return err + } + + return cmd.Run(ctx, devPodConfig, client, src, dst, log.Default) + }, + ValidArgsFunction: func(rootCmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if strings.Contains(toComplete, ":") { + return nil, cobra.ShellCompDirectiveNoFileComp + } + workspaces, _ := completion.GetWorkspaceSuggestions(rootCmd, cmd.Context, cmd.Provider, nil, toComplete, log.Default) + for i := range workspaces { + workspaces[i] += ":" + } + return workspaces, cobra.ShellCompDirectiveDefault | cobra.ShellCompDirectiveNoSpace + }, + } + + cpCmd.Flags().StringVar(&cmd.User, "user", "", "The user in the container to copy as, which owns the files copied into it. Defaults to the workspace's user") + return cpCmd +} + +// cpLocation is a path given to devpod cp, in a workspace if workspace isn't empty. +type cpLocation struct { + workspace string + path string +} + +func (l cpLocation) String() string { + if l.workspace == "" { + return l.path + } + return l.workspace + ":" + l.path +} + +// parseCpLocation splits WORKSPACE:PATH, like scp: an argument is a local path if it has no colon, has a slash +// before its first colon, or (on Windows) starts with a drive letter. +func parseCpLocation(arg, goos string) (cpLocation, error) { + workspace, p, found := strings.Cut(arg, ":") + if !found || strings.ContainsAny(workspace, `/\`) || (goos == "windows" && isDriveLetter(workspace)) { + return cpLocation{path: arg}, nil + } + if workspace == "" { + return cpLocation{}, fmt.Errorf("%s has no workspace before the colon: write a path in a workspace as WORKSPACE:PATH, or a local path that starts with a colon as ./%s", arg, arg) + } + return cpLocation{workspace: workspace, path: p}, nil +} + +func isDriveLetter(s string) bool { + return len(s) == 1 && ('a' <= s[0] && s[0] <= 'z' || 'A' <= s[0] && s[0] <= 'Z') +} + +// Run copies src to dst, one of which is in the workspace client is for +func (cmd *CpCmd) Run( + ctx context.Context, + devPodConfig *config.Config, + client client2.WorkspaceClient, + src, dst cpLocation, + log log.Logger, +) error { + workspaceConfig := client.WorkspaceConfig() + user := cmd.User + if user == "" { + var err error + user, err = devssh.GetUser(workspaceConfig.ID, workspaceConfig.SSHConfigPath) + if err != nil { + return err + } + } + + workdir, err := devssh.GetWorkdir(workspaceConfig.ID, workspaceConfig.SSHConfigPath) + if err != nil { + return err + } + if workdir == "" { + workdir = path.Join("/workspaces", client.Workspace()) + } + + // lock the workspace as long as we init the connection + err = client.Lock(ctx) + if err != nil { + return err + } + defer client.Unlock() + + err = startWait(ctx, client, false, log) + if err != nil { + return err + } + + return tunnel.NewContainerTunnel(client, log.ErrorStreamOnly()). + Run(ctx, func(ctx context.Context, containerClient *ssh.Client) error { + // we have a connection to the container, make sure others can connect as well + client.Unlock() + + return withContainerSFTP(ctx, containerClient, user, workdir, log, func(sftpClient *sftp.Client) error { + var srcFS, dstFS filetransfer.FS = filetransfer.LocalFS{}, filetransfer.LocalFS{} + remote := filetransfer.SFTPFS{Client: sftpClient, WorkDir: workdir} + if src.workspace != "" { + srcFS = remote + } else { + dstFS = remote + } + + stats, err := filetransfer.Copy(srcFS, src.path, dstFS, dst.path, log) + if err != nil { + var remoteErr *filetransfer.RemoteError + if user != "root" && errors.As(err, &remoteErr) && errors.Is(remoteErr, fs.ErrPermission) { + return fmt.Errorf("copy %s to %s as user %s: %w (use --user root to copy as root)", src, dst, user, err) + } + return fmt.Errorf("copy %s to %s: %w", src, dst, err) + } + log.Donef("Copied %s from %s to %s", stats, src, dst) + return nil + }) + }, devPodConfig, nil) +} + +// withContainerSFTP starts an SSH server in the container as user, as devpod ssh does, and calls fn with an SFTP +// client connected to it. +func withContainerSFTP( + ctx context.Context, + containerClient *ssh.Client, + user, workdir string, + log log.Logger, + fn func(*sftp.Client) error, +) error { + command := fmt.Sprintf("'%s' helper ssh-server --stdio --workdir '%s'", agent.ContainerDevPodHelperLocation, workdir) + if log.GetLevel() == logrus.DebugLevel { + command += " --debug" + } + if user != "" && user != "root" { + command = fmt.Sprintf("su -c \"%s\" '%s'", command, user) + } + + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return err + } + defer stdoutReader.Close() + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return err + } + defer stdinReader.Close() + defer stdinWriter.Close() + + writer := log.ErrorStreamOnly().Writer(logrus.InfoLevel, false) + defer writer.Close() + + serverErr := make(chan error, 1) + go func() { + defer stdoutWriter.Close() + log.Debugf("Run SSH server in container: %s", command) + serverErr <- devssh.Run(ctx, containerClient, command, stdinReader, stdoutWriter, writer, nil) + }() + + sshClient, err := devssh.StdioClientWithUser(stdoutReader, stdinWriter, user, false) + if err != nil { + select { + case runErr := <-serverErr: + if runErr != nil { + return fmt.Errorf("start SSH server in container: %w", runErr) + } + default: + } + return fmt.Errorf("connect to SSH server in container: %w", err) + } + defer sshClient.Close() + + sftpClient, err := sftp.NewClient(sshClient, sftp.UseConcurrentWrites(true)) + if err != nil { + return fmt.Errorf("start SFTP session: %w", err) + } + defer sftpClient.Close() + + return fn(sftpClient) +} diff --git a/cmd/cp_test.go b/cmd/cp_test.go new file mode 100644 index 000000000..465f3a170 --- /dev/null +++ b/cmd/cp_test.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "testing" +) + +func TestParseCpLocation(t *testing.T) { + tests := []struct { + arg string + goos string + workspace string + path string + err bool + }{ + {arg: "my-file.txt", path: "my-file.txt"}, + {arg: "./here/file-here.txt", path: "./here/file-here.txt"}, + {arg: "foo:", workspace: "foo", path: ""}, + {arg: "foo:file-in-container.txt", workspace: "foo", path: "file-in-container.txt"}, + {arg: "foo:/usr/local/", workspace: "foo", path: "/usr/local/"}, + {arg: "foo:a:b", workspace: "foo", path: "a:b"}, + {arg: "./notes:2024.txt", path: "./notes:2024.txt"}, + {arg: "dir/notes:2024.txt", path: "dir/notes:2024.txt"}, + {arg: `dir\notes:2024.txt`, path: `dir\notes:2024.txt`}, + {arg: ":file", err: true}, + {arg: "c:file", goos: "linux", workspace: "c", path: "file"}, + {arg: `C:\Users\me\file.txt`, goos: "windows", path: `C:\Users\me\file.txt`}, + {arg: "C:file.txt", goos: "windows", path: "C:file.txt"}, + {arg: "foo:file.txt", goos: "windows", workspace: "foo", path: "file.txt"}, + } + for _, tc := range tests { + t.Run(tc.arg, func(t *testing.T) { + goos := tc.goos + if goos == "" { + goos = "linux" + } + location, err := parseCpLocation(tc.arg, goos) + if tc.err { + if err == nil { + t.Fatalf("expected an error, got %+v", location) + } + return + } + if err != nil { + t.Fatal(err) + } + if location.workspace != tc.workspace || location.path != tc.path { + t.Errorf("expected workspace %q and path %q, got %+v", tc.workspace, tc.path, location) + } + }) + } +} diff --git a/cmd/helper/helper.go b/cmd/helper/helper.go index 89a290732..f33d3c35c 100644 --- a/cmd/helper/helper.go +++ b/cmd/helper/helper.go @@ -24,6 +24,7 @@ func NewHelperCmd(globalFlags *flags.GlobalFlags) *cobra.Command { helperCmd.AddCommand(json.NewJSONCmd(globalFlags)) helperCmd.AddCommand(strings.NewStringsCmd(globalFlags)) helperCmd.AddCommand(NewSSHServerCmd(globalFlags)) + helperCmd.AddCommand(NewSFTPServerCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceNameCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceUIDCmd(globalFlags)) helperCmd.AddCommand(NewGetWorkspaceConfigCommand(globalFlags)) diff --git a/cmd/helper/sftp_server.go b/cmd/helper/sftp_server.go new file mode 100644 index 000000000..a4bc2db6b --- /dev/null +++ b/cmd/helper/sftp_server.go @@ -0,0 +1,40 @@ +package helper + +import ( + "errors" + "io" + "os" + + "github.com/curityio/devpod/cmd/flags" + "github.com/curityio/devpod/pkg/stdio" + "github.com/pkg/sftp" + "github.com/spf13/cobra" +) + +// NewSFTPServerCmd creates a command that serves SFTP over stdin and stdout, like OpenSSH's sftp-server. The SSH +// server runs it for SFTP sessions in the user's home directory, which relative paths are relative to. +func NewSFTPServerCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + return &cobra.Command{ + Use: "sftp-server", + Short: "Serves SFTP over stdin and stdout", + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, _ []string) error { + // stdout is the SFTP stream, so nothing else may be written to it + var options []sftp.ServerOption + if globalFlags.Debug { + options = append(options, sftp.WithDebug(os.Stderr)) + } + server, err := sftp.NewServer(stdio.NewStdioStream(os.Stdin, os.Stdout, false, 0), options...) + if err != nil { + return err + } + defer server.Close() + + err = server.Serve() + if errors.Is(err, io.EOF) { + return nil + } + return err + }, + } +} diff --git a/cmd/root.go b/cmd/root.go index 5d8800ebf..9662a8c43 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -119,6 +119,7 @@ func BuildRoot() *cobra.Command { rootCmd.AddCommand(NewUpCmd(globalFlags)) rootCmd.AddCommand(NewDeleteCmd(globalFlags)) rootCmd.AddCommand(NewSSHCmd(globalFlags)) + rootCmd.AddCommand(NewCpCmd(globalFlags)) rootCmd.AddCommand(NewVersionCmd()) rootCmd.AddCommand(NewStopCmd(globalFlags)) rootCmd.AddCommand(NewListCmd(globalFlags)) diff --git a/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx b/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx index 250ae9d4b..e6d134fa2 100644 --- a/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx +++ b/docs/pages/developing-in-workspaces/connect-to-a-workspace.mdx @@ -88,6 +88,34 @@ Optionally you can also define a command to run: devpod ssh my-workspace --command "echo Hello World" ``` +### Copy Files + +`devpod cp` copies files and directories between your machine and a workspace. A path in the workspace is +written as `WORKSPACE_NAME:PATH`, like with `scp`. A relative path is relative to the workspace folder, and an +empty one is the workspace folder itself: +``` +# copy a file into the workspace folder +devpod cp my-file.txt my-workspace: + +# copy a file to a path relative to the workspace folder, under another name +devpod cp ./here/file-here.txt my-workspace:file-in-container.txt + +# copy a directory into an absolute path in the container +devpod cp ./temp/ my-workspace:/opt/ + +# copy a file from the workspace to the current directory +devpod cp my-workspace:/etc/os-release . +``` + +Like `docker cp`, a file or directory is copied into the destination if that is an existing directory, and to the +destination otherwise. A destination that ends with `/` must be a directory, and is created if it's missing. A +source that ends with `/.` copies a directory's contents instead of the directory. Directories are copied with +everything in them, keeping permissions, modification times and symlinks. + +Files are copied as the workspace's user, who owns the files copied into the workspace. To write where only root +can, such as `/usr/local`, add `--user root`. To copy a local file whose name has a colon in it, start its path +with `./`. + ## IDE Commands This section shows additional commands to configure DevPod's behavior when opening a workspace. diff --git a/go.mod b/go.mod index f869b4527..9b826e36f 100644 --- a/go.mod +++ b/go.mod @@ -164,7 +164,7 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect - github.com/docker/go-units v0.5.0 // indirect + github.com/docker/go-units v0.5.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -188,7 +188,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/sync v0.12.0 golang.org/x/text v0.23.0 golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.29.0 // indirect diff --git a/pkg/filetransfer/copy.go b/pkg/filetransfer/copy.go new file mode 100644 index 000000000..c2277f5fc --- /dev/null +++ b/pkg/filetransfer/copy.go @@ -0,0 +1,351 @@ +package filetransfer + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "sync" + + "github.com/curityio/devpod/pkg/log" + "github.com/docker/go-units" + "golang.org/x/sync/errgroup" +) + +// concurrency is how many files are copied at once, which hides the latency of each SFTP request. +const concurrency = 8 + +// Stats says what Copy copied. +type Stats struct { + Files int + Bytes int64 +} + +// String summarizes stats, such as "3 files (1.2kB)". +func (s Stats) String() string { + files := "files" + if s.Files == 1 { + files = "file" + } + return fmt.Sprintf("%d %s (%s)", s.Files, files, units.HumanSize(float64(s.Bytes))) +} + +// Copy copies srcPath on src to dstPath on dst, following the rules of `docker cp`: +// +// - A file is copied into dstPath if that is a directory, and to dstPath otherwise. +// - A directory is copied into dstPath if that is a directory, and to dstPath otherwise. A srcPath that ends with +// "/." copies the directory's contents into dstPath instead. +// - A dstPath that ends with a separator must be a directory, and is created, with any missing parents, if it +// doesn't exist. +// +// Within a directory, symlinks are copied as symlinks, and files that aren't regular files, directories or symlinks +// are skipped. The permission bits and modification times are kept. Existing files are replaced: each file is +// written to a temporary file next to it and renamed over it, so that copying a file onto itself (through a mount) +// doesn't truncate it, and read-only files can be replaced. +func Copy(src FS, srcPath string, dst FS, dstPath string, log log.Logger) (Stats, error) { + contentsOnly := endsWithDot(src, srcPath) + dstMustBeDir := dstPath == "" || dst.IsSeparator(dstPath[len(dstPath)-1]) + + srcPath, err := src.Abs(srcPath) + if err != nil { + return Stats{}, err + } + dstPath, err = dst.Abs(dstPath) + if err != nil { + return Stats{}, err + } + + srcInfo, err := src.Stat(srcPath) + if err != nil { + return Stats{}, err + } + dstInfo, err := dst.Stat(dstPath) + dstExists := err == nil + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return Stats{}, err + } + + if dstExists && !dstInfo.IsDir() && (dstMustBeDir || srcInfo.IsDir()) { + return Stats{}, fmt.Errorf("%s is not a directory", dstPath) + } + target := dstPath + switch { + case dstExists && dstInfo.IsDir(): + if !srcInfo.IsDir() || !contentsOnly { + target = dst.Join(dstPath, src.Base(srcPath)) + } + case dstMustBeDir && srcInfo.IsDir(): + // dstPath becomes the copy of the directory, so only its parents are created here + err = mkdirAll(dst, dst.Dir(dstPath)) + if err != nil { + return Stats{}, err + } + case dstMustBeDir: + err = mkdirAll(dst, dstPath) + if err != nil { + return Stats{}, err + } + target = dst.Join(dstPath, src.Base(srcPath)) + } + + // List everything before writing anything, so that copying a directory into itself (which is possible when the + // container's workspace folder is a local folder mounted into it) doesn't go on forever. + c := &copier{src: src, dst: dst, log: log, created: map[string]bool{}} + entries, err := c.walk(srcPath, srcInfo, target) + if err != nil { + return Stats{}, err + } + + // Directories first, parents before children, so that files can then be copied in any order. + for _, e := range entries { + if e.info.IsDir() { + err = c.copyDir(e) + if err != nil { + return c.stats, err + } + } + } + + group, ctx := errgroup.WithContext(context.Background()) + group.SetLimit(concurrency) + for _, e := range entries { + if !e.info.IsDir() { + group.Go(func() error { + // stop at the first failure + if ctx.Err() != nil { + return nil + } + return c.copy(e) + }) + } + } + err = group.Wait() + if err != nil { + return c.stats, err + } + + // Set directories' metadata last, as writing into them changes their modification time, and a read-only + // directory can't be written into. Children go before their parents. + for i := len(entries) - 1; i >= 0; i-- { + if entries[i].info.IsDir() { + c.setDirMetadata(entries[i]) + } + } + + return c.stats, nil +} + +func mkdirAll(f FS, p string) error { + err := f.MkdirAll(p) + if err != nil { + return fmt.Errorf("create directory %s: %w", p, err) + } + return nil +} + +// endsWithDot reports whether p ends with a separator followed by ".", which means a directory's contents. +func endsWithDot(f FS, p string) bool { + return len(p) >= 2 && p[len(p)-1] == '.' && f.IsSeparator(p[len(p)-2]) +} + +type entry struct { + info fs.FileInfo + srcPath, dstPath string + // dstDir is the directory dstPath is in. + dstDir string +} + +type copier struct { + src, dst FS + log log.Logger + + // created holds the directories this copy created, which can't have anything in them yet. It's only written + // to before files are copied concurrently. + created map[string]bool + + mu sync.Mutex + stats Stats +} + +// walk lists srcPath and everything in it, skipping what can't be named on the destination. +func (c *copier) walk(srcPath string, srcInfo fs.FileInfo, target string) ([]entry, error) { + entries := []entry{{info: srcInfo, srcPath: srcPath, dstPath: target, dstDir: c.dst.Dir(target)}} + for i := 0; i < len(entries); i++ { + parent := entries[i] + if !parent.info.IsDir() { + continue + } + children, err := c.src.ReadDir(parent.srcPath) + if err != nil { + return nil, fmt.Errorf("read directory %s: %w", parent.srcPath, err) + } + for _, child := range children { + name := child.Name() + if !c.src.ValidName(name) || !c.dst.ValidName(name) { + c.log.Warnf("Skipping %q in %s, which isn't a valid file name on the destination", name, parent.srcPath) + continue + } + entries = append(entries, entry{ + info: child, + srcPath: c.src.Join(parent.srcPath, name), + dstPath: c.dst.Join(parent.dstPath, name), + dstDir: parent.dstPath, + }) + } + } + return entries, nil +} + +func (c *copier) copy(e entry) error { + mode := e.info.Mode() + switch { + case mode&fs.ModeSymlink != 0: + return c.copySymlink(e) + case mode.IsRegular(): + return c.copyFile(e) + default: + c.log.Warnf("Skipping %s, which is not a regular file, directory or symlink", e.srcPath) + return nil + } +} + +func (c *copier) copyDir(e entry) error { + if !c.created[e.dstDir] { + existing, err := c.dst.Lstat(e.dstPath) + switch { + case err == nil && !existing.IsDir(): + return fmt.Errorf("%s is not a directory", e.dstPath) + case err == nil: + // a read-only directory, maybe from an earlier copy, has to be writable until its metadata is set + if existing.Mode().Perm()&0o200 == 0 { + _ = c.dst.Chmod(e.dstPath, existing.Mode().Perm()|0o700) + } + return nil + case !errors.Is(err, fs.ErrNotExist): + return err + } + } + + c.log.Debugf("Creating directory %s", e.dstPath) + err := c.dst.Mkdir(e.dstPath) + if err != nil { + return fmt.Errorf("create directory %s: %w", e.dstPath, err) + } + c.created[e.dstPath] = true + return nil +} + +func (c *copier) copySymlink(e entry) error { + target, err := c.src.Readlink(e.srcPath) + if err != nil { + return fmt.Errorf("read symlink %s: %w", e.srcPath, err) + } + target = c.dst.FromSlash(c.src.ToSlash(target)) + + if !c.created[e.dstDir] { + err = c.removeExisting(e.dstPath) + if err != nil { + return err + } + } + c.log.Debugf("Creating symlink %s -> %s", e.dstPath, target) + err = c.dst.Symlink(target, e.dstPath) + if errors.Is(err, ErrSymlinkNotPermitted) { + c.log.Warnf("Skipping symlink %s -> %s: %v", e.srcPath, target, err) + return nil + } else if err != nil { + return fmt.Errorf("create symlink %s: %w", e.dstPath, err) + } + c.count(0) + return nil +} + +func (c *copier) copyFile(e entry) (err error) { + c.log.Debugf("Copying %s to %s", e.srcPath, e.dstPath) + in, err := c.src.Open(e.srcPath) + if err != nil { + return err + } + defer in.Close() + + tmp := c.dst.Join(e.dstDir, "."+c.dst.Base(e.dstPath)+".devpod-cp-"+randomSuffix()) + out, err := c.dst.CreateNew(tmp) + if err != nil { + return fmt.Errorf("create %s: %w", tmp, err) + } + defer func() { + if err != nil { + _ = c.dst.Remove(tmp) + } + }() + + n, err := io.Copy(out, in) + closeErr := out.Close() + if err != nil { + return fmt.Errorf("copy %s to %s: %w", e.srcPath, e.dstPath, err) + } else if closeErr != nil { + return fmt.Errorf("write %s: %w", tmp, closeErr) + } + + err = c.dst.Chmod(tmp, e.info.Mode().Perm()) + if err != nil { + return fmt.Errorf("set permissions of %s: %w", tmp, err) + } + if chtimesErr := c.dst.Chtimes(tmp, e.info.ModTime()); chtimesErr != nil { + c.log.Warnf("Set modification time of %s: %v", e.dstPath, chtimesErr) + } + + err = c.dst.Rename(tmp, e.dstPath) + if err != nil { + if existing, statErr := c.dst.Lstat(e.dstPath); statErr == nil && existing.IsDir() { + return fmt.Errorf("%s is a directory", e.dstPath) + } + return fmt.Errorf("replace %s: %w", e.dstPath, err) + } + + c.count(n) + return nil +} + +func (c *copier) count(bytes int64) { + c.mu.Lock() + defer c.mu.Unlock() + c.stats.Files++ + c.stats.Bytes += bytes +} + +func (c *copier) removeExisting(p string) error { + existing, err := c.dst.Lstat(p) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if existing.IsDir() { + return fmt.Errorf("%s is a directory", p) + } + return c.dst.Remove(p) +} + +// setDirMetadata copies a directory's permission bits and modification time. It only warns of failures: an existing +// directory that belongs to another user can be written to, but not changed in these ways. +func (c *copier) setDirMetadata(e entry) { + err := c.dst.Chmod(e.dstPath, e.info.Mode().Perm()) + if err != nil { + c.log.Warnf("Set permissions of %s: %v", e.dstPath, err) + } + err = c.dst.Chtimes(e.dstPath, e.info.ModTime()) + if err != nil { + c.log.Warnf("Set modification time of %s: %v", e.dstPath, err) + } +} + +func randomSuffix() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/pkg/filetransfer/copy_test.go b/pkg/filetransfer/copy_test.go new file mode 100644 index 000000000..9aba939b7 --- /dev/null +++ b/pkg/filetransfer/copy_test.go @@ -0,0 +1,469 @@ +package filetransfer + +import ( + "errors" + "io/fs" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/curityio/devpod/pkg/log" + "github.com/google/go-cmp/cmp" + "github.com/pkg/sftp" +) + +// tree describes a directory's contents: a file's content, "dir/" for a directory, or "-> target" for a symlink. +type tree map[string]string + +func writeTree(t *testing.T, root string, files tree) { + t.Helper() + for name, content := range files { + p := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + var err error + switch { + case content == "dir/": + err = os.MkdirAll(p, 0o755) + case len(content) > 3 && content[:3] == "-> ": + err = os.Symlink(content[3:], p) + default: + err = os.WriteFile(p, []byte(content), 0o644) + } + if err != nil { + t.Fatal(err) + } + } +} + +func readTree(t *testing.T, root string) tree { + t.Helper() + files := tree{} + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil || p == root { + return err + } + rel, _ := filepath.Rel(root, p) + rel = filepath.ToSlash(rel) + switch { + case d.Type()&fs.ModeSymlink != 0: + target, err := os.Readlink(p) + files[rel] = "-> " + target + return err + case d.IsDir(): + files[rel] = "dir/" + default: + content, err := os.ReadFile(p) + files[rel] = string(content) + return err + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return files +} + +// newSFTPFS serves the local file system over SFTP in-process, the way the container's SSH server does. +func newSFTPFS(t *testing.T, workDir string) SFTPFS { + t.Helper() + serverConn, clientConn := net.Pipe() + server, err := sftp.NewServer(serverConn) + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve() }() + client, err := sftp.NewClientPipe(clientConn, clientConn, sftp.UseConcurrentWrites(true)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = client.Close() + _ = server.Close() + }) + return SFTPFS{Client: client, WorkDir: workDir} +} + +type copyCase struct { + name string + src tree + dst tree + srcArg string // relative to the source root, with / separators + dstArg string // relative to the destination root, with / separators + expected tree // the destination root afterwards + err bool + symlinks bool +} + +var copyCases = []copyCase{ + { + name: "file into existing directory", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "", + expected: tree{"a.txt": "a"}, + }, + { + name: "file to new name", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "b.txt", + expected: tree{"b.txt": "a"}, + }, + { + name: "file into missing directory written with trailing slash", + src: tree{"a.txt": "a"}, + srcArg: "a.txt", + dstArg: "new/dir/", + expected: tree{"new": "dir/", "new/dir": "dir/", "new/dir/a.txt": "a"}, + }, + { + name: "file overwrites existing file", + src: tree{"a.txt": "new"}, + dst: tree{"a.txt": "old content"}, + srcArg: "a.txt", + dstArg: "a.txt", + expected: tree{"a.txt": "new"}, + }, + { + name: "file onto path with trailing slash that is a file", + src: tree{"a.txt": "a"}, + dst: tree{"b.txt": "b"}, + srcArg: "a.txt", + dstArg: "b.txt/", + err: true, + }, + { + name: "directory into existing directory", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y", "temp/empty": "dir/"}, + dst: tree{"existing.txt": "e"}, + srcArg: "temp/", + dstArg: "", + expected: tree{"existing.txt": "e", "temp": "dir/", "temp/x.txt": "x", "temp/sub": "dir/", "temp/sub/y.txt": "y", "temp/empty": "dir/"}, + }, + { + name: "directory to missing nested directory written with trailing slash", + src: tree{"temp/x.txt": "x"}, + srcArg: "temp", + dstArg: "a/b/", + expected: tree{"a": "dir/", "a/b": "dir/", "a/b/x.txt": "x"}, + }, + { + name: "directory to new path", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y"}, + srcArg: "temp", + dstArg: "copy", + expected: tree{"copy": "dir/", "copy/x.txt": "x", "copy/sub": "dir/", "copy/sub/y.txt": "y"}, + }, + { + name: "directory contents merged into existing directory", + src: tree{"temp/x.txt": "x", "temp/sub/y.txt": "y"}, + dst: tree{"sub/z.txt": "z", "x.txt": "old"}, + srcArg: "temp/.", + dstArg: "", + expected: tree{"x.txt": "x", "sub": "dir/", "sub/y.txt": "y", "sub/z.txt": "z"}, + }, + { + name: "directory onto file", + src: tree{"temp/x.txt": "x"}, + dst: tree{"file": "f"}, + srcArg: "temp", + dstArg: "file", + err: true, + }, + { + name: "symlinks are copied as symlinks", + src: tree{"temp/x.txt": "x", "temp/link": "-> x.txt", "temp/dangling": "-> nowhere"}, + dst: tree{"temp/link": "replaced"}, + srcArg: "temp", + dstArg: "", + expected: tree{"temp": "dir/", "temp/x.txt": "x", "temp/link": "-> x.txt", "temp/dangling": "-> nowhere"}, + symlinks: true, + }, +} + +func runCopyCases(t *testing.T, newDst func(t *testing.T, root string) FS, toDstArg func(root, rel string) string) { + for _, tc := range copyCases { + t.Run(tc.name, func(t *testing.T) { + if tc.symlinks && runtime.GOOS == "windows" { + t.Skip("creating symlinks needs privileges on Windows") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tc.src) + writeTree(t, dstRoot, tc.dst) + + srcArg := filepath.Join(srcRoot, filepath.FromSlash(tc.srcArg)) + if len(tc.srcArg) > 0 && tc.srcArg[len(tc.srcArg)-1] == '/' { + srcArg += string(os.PathSeparator) + } else if filepath.Base(tc.srcArg) == "." { + srcArg = filepath.Join(srcRoot, filepath.FromSlash(filepath.Dir(tc.srcArg))) + string(os.PathSeparator) + "." + } + + _, err := Copy(LocalFS{}, srcArg, newDst(t, dstRoot), toDstArg(dstRoot, tc.dstArg), log.Discard) + if tc.err { + if err == nil { + t.Fatal("expected an error") + } + return + } + if err != nil { + t.Fatal(err) + } + + expected := tree{} + for name, content := range tc.dst { + expected[name] = content + } + for name, content := range tc.expected { + expected[name] = content + } + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + }) + } +} + +func TestCopyLocal(t *testing.T) { + runCopyCases(t, + func(t *testing.T, root string) FS { return LocalFS{} }, + func(root, rel string) string { + p := filepath.Join(root, filepath.FromSlash(rel)) + if rel == "" || rel[len(rel)-1] == '/' { + p += string(os.PathSeparator) + } + return p + }) +} + +func TestCopyToSFTP(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + // relative to the work dir, as in a workspace + runCopyCases(t, + func(t *testing.T, root string) FS { return newSFTPFS(t, root) }, + func(root, rel string) string { return rel }) +} + +func TestCopyFromSFTP(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + remoteRoot, localRoot := t.TempDir(), t.TempDir() + writeTree(t, remoteRoot, tree{"temp/x.txt": "x", "temp/sub/y.txt": "y", "temp/link": "-> x.txt"}) + + stats, err := Copy(newSFTPFS(t, remoteRoot), "temp", LocalFS{}, localRoot, log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"temp": "dir/", "temp/x.txt": "x", "temp/sub": "dir/", "temp/sub/y.txt": "y", "temp/link": "-> x.txt"} + if diff := cmp.Diff(expected, readTree(t, localRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + if stats.Files != 3 || stats.Bytes != 2 { + t.Errorf("expected 3 files of 2 bytes, got %+v", stats) + } +} + +func TestCopyKeepsModeAndModTime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows has no executable bit") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + script := filepath.Join(srcRoot, "dir", "script.sh") + writeTree(t, srcRoot, tree{"dir/script.sh": "#!/bin/sh"}) + if err := os.Chmod(script, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(srcRoot, "dir"), 0o700); err != nil { + t.Fatal(err) + } + mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + for _, p := range []string{script, filepath.Join(srcRoot, "dir")} { + if err := os.Chtimes(p, mtime, mtime); err != nil { + t.Fatal(err) + } + } + + _, err := Copy(LocalFS{}, filepath.Join(srcRoot, "dir"), newSFTPFS(t, dstRoot), "", log.Discard) + if err != nil { + t.Fatal(err) + } + + for p, mode := range map[string]fs.FileMode{"dir/script.sh": 0o750, "dir": 0o700 | fs.ModeDir} { + info, err := os.Stat(filepath.Join(dstRoot, p)) + if err != nil { + t.Fatal(err) + } + if info.Mode() != mode { + t.Errorf("%s: expected mode %v, got %v", p, mode, info.Mode()) + } + if !info.ModTime().Equal(mtime) { + t.Errorf("%s: expected modification time %v, got %v", p, mtime, info.ModTime()) + } + } +} + +func TestCopyDirectoryIntoItself(t *testing.T) { + root := t.TempDir() + writeTree(t, root, tree{"dir/x.txt": "x"}) + + _, err := Copy(LocalFS{}, filepath.Join(root, "dir"), LocalFS{}, filepath.Join(root, "dir", "inner")+string(os.PathSeparator), log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"dir": "dir/", "dir/x.txt": "x", "dir/inner": "dir/", "dir/inner/x.txt": "x"} + if diff := cmp.Diff(expected, readTree(t, root)); diff != "" { + t.Errorf("unexpected tree (-want +got):\n%s", diff) + } +} + +func TestCopyFileOntoItself(t *testing.T) { + // as when the workspace folder is the local folder mounted into the container + root := t.TempDir() + writeTree(t, root, tree{"a.txt": "content", "dir/b.txt": "b"}) + + // devpod cp a.txt foo: and devpod cp dir foo:, from the workspace folder + for _, p := range []string{"a.txt", "dir"} { + _, err := Copy(LocalFS{}, filepath.Join(root, p), LocalFS{}, root+string(os.PathSeparator), log.Discard) + if err != nil { + t.Fatal(err) + } + } + + expected := tree{"a.txt": "content", "dir": "dir/", "dir/b.txt": "b"} + if diff := cmp.Diff(expected, readTree(t, root)); diff != "" { + t.Errorf("unexpected tree (-want +got):\n%s", diff) + } +} + +func TestCopyReadOnlyTreeTwice(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows has no read-only directories") + } + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tree{"repo/objects/ab/cdef": "object"}) + for p, mode := range map[string]fs.FileMode{"repo/objects/ab/cdef": 0o444, "repo/objects/ab": 0o555} { + if err := os.Chmod(filepath.Join(srcRoot, p), mode); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + // let t.TempDir remove them + for _, root := range []string{srcRoot, dstRoot} { + _ = os.Chmod(filepath.Join(root, "repo/objects/ab"), 0o755) + } + }) + + for i := 0; i < 2; i++ { + _, err := Copy(LocalFS{}, filepath.Join(srcRoot, "repo"), newSFTPFS(t, dstRoot), "", log.Discard) + if err != nil { + t.Fatalf("copy %d: %v", i+1, err) + } + } + + expected := tree{"repo": "dir/", "repo/objects": "dir/", "repo/objects/ab": "dir/", "repo/objects/ab/cdef": "object"} + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + info, err := os.Stat(filepath.Join(dstRoot, "repo/objects/ab")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o555 { + t.Errorf("expected the directory to be read-only again, got %v", info.Mode()) + } +} + +// strictFS rejects names that start with "bad", as Windows rejects names with a backslash or colon. +type strictFS struct{ LocalFS } + +func (f strictFS) ValidName(name string) bool { + return f.LocalFS.ValidName(name) && !strings.HasPrefix(name, "bad") +} + +func TestCopySkipsInvalidNames(t *testing.T) { + srcRoot, dstRoot := t.TempDir(), t.TempDir() + writeTree(t, srcRoot, tree{"temp/ok.txt": "ok", "temp/bad.txt": "bad", "temp/bad-dir/x.txt": "x"}) + + stats, err := Copy(LocalFS{}, filepath.Join(srcRoot, "temp"), strictFS{}, dstRoot, log.Discard) + if err != nil { + t.Fatal(err) + } + + expected := tree{"temp": "dir/", "temp/ok.txt": "ok"} + if diff := cmp.Diff(expected, readTree(t, dstRoot)); diff != "" { + t.Errorf("unexpected destination (-want +got):\n%s", diff) + } + if stats.Files != 1 { + t.Errorf("expected 1 file, got %+v", stats) + } +} + +func TestValidName(t *testing.T) { + for _, name := range []string{"", ".", "..", "a/b"} { + if (SFTPFS{}).ValidName(name) || (LocalFS{}).ValidName(name) { + t.Errorf("expected %q to be invalid", name) + } + } + for _, name := range []string{"a.txt", ".hidden", "..."} { + if !(SFTPFS{}).ValidName(name) || !(LocalFS{}).ValidName(name) { + t.Errorf("expected %q to be valid", name) + } + } + for _, name := range []string{`..\..\evil`, "a:b"} { + if !(SFTPFS{}).ValidName(name) { + t.Errorf("expected %q to be valid in a container", name) + } + if (LocalFS{}).ValidName(name) == (runtime.GOOS == "windows") { + t.Errorf("expected %q to be valid locally only if not on Windows", name) + } + } +} + +func TestSFTPCreateNewIsPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + root := t.TempDir() + remote := newSFTPFS(t, root) + f, err := remote.CreateNew(filepath.Join(root, "secret")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + info, err := os.Stat(filepath.Join(root, "secret")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("expected a new file to be 0600 until it's written, got %v", info.Mode()) + } +} + +func TestSFTPErrorsAreRemote(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the SFTP server is only used in Linux containers") + } + root := t.TempDir() + _, err := Copy(newSFTPFS(t, root), "missing.txt", LocalFS{}, t.TempDir(), log.Discard) + var remoteErr *RemoteError + if !errors.As(err, &remoteErr) || !errors.Is(err, fs.ErrNotExist) { + t.Errorf("expected a remote not-exist error, got %v", err) + } + + _, err = Copy(LocalFS{}, filepath.Join(root, "missing.txt"), newSFTPFS(t, root), "", log.Discard) + if errors.As(err, &remoteErr) || !errors.Is(err, fs.ErrNotExist) { + t.Errorf("expected a local not-exist error, got %v", err) + } +} diff --git a/pkg/filetransfer/fs.go b/pkg/filetransfer/fs.go new file mode 100644 index 000000000..f91b85d3c --- /dev/null +++ b/pkg/filetransfer/fs.go @@ -0,0 +1,250 @@ +// Package filetransfer copies files and directories between two file systems, such as the local one and a dev +// container's, reached over SFTP. +package filetransfer + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/pkg/sftp" +) + +// ErrSymlinkNotPermitted is returned by Symlink on Windows when the user may not create symlinks, which needs +// Developer Mode or an administrator. +var ErrSymlinkNotPermitted = errors.New("creating symlinks isn't permitted") + +// FS is a file system that files can be copied from or to. +type FS interface { + // Abs makes p absolute. + Abs(p string) (string, error) + Join(elem ...string) string + Base(p string) string + Dir(p string) string + IsSeparator(c byte) bool + // ValidName reports whether name can be used as a single file name, so that it can't refer to another + // directory. + ValidName(name string) bool + // ToSlash and FromSlash convert a symlink target between this file system's separator and /. + ToSlash(p string) string + FromSlash(p string) string + + Stat(p string) (fs.FileInfo, error) + Lstat(p string) (fs.FileInfo, error) + // ReadDir lists a directory without following symlinks. + ReadDir(p string) ([]fs.FileInfo, error) + Readlink(p string) (string, error) + + Open(p string) (io.ReadCloser, error) + // CreateNew creates a file that doesn't exist yet, which only its owner can read and write. + CreateNew(p string) (io.WriteCloser, error) + // Rename moves oldPath to newPath, replacing newPath if it exists and isn't a directory. + Rename(oldPath, newPath string) error + Mkdir(p string) error + MkdirAll(p string) error + Remove(p string) error + Symlink(target, p string) error + Chmod(p string, mode fs.FileMode) error + Chtimes(p string, mtime time.Time) error +} + +// LocalFS is the file system of the machine DevPod runs on. +type LocalFS struct{} + +var _ FS = LocalFS{} + +func (LocalFS) Abs(p string) (string, error) { return filepath.Abs(p) } +func (LocalFS) Join(elem ...string) string { return filepath.Join(elem...) } +func (LocalFS) Base(p string) string { return filepath.Base(p) } +func (LocalFS) Dir(p string) string { return filepath.Dir(p) } +func (LocalFS) IsSeparator(c byte) bool { return os.IsPathSeparator(c) } +func (LocalFS) ToSlash(p string) string { return filepath.ToSlash(p) } +func (LocalFS) FromSlash(p string) string { return filepath.FromSlash(p) } +func (LocalFS) Readlink(p string) (string, error) { return os.Readlink(p) } +func (LocalFS) Open(p string) (io.ReadCloser, error) { return os.Open(p) } +func (LocalFS) Mkdir(p string) error { return os.Mkdir(p, 0o755) } +func (LocalFS) MkdirAll(p string) error { return os.MkdirAll(p, 0o755) } +func (LocalFS) Remove(p string) error { return os.Remove(p) } +func (LocalFS) Chmod(p string, mode fs.FileMode) error { return os.Chmod(p, mode) } +func (LocalFS) Chtimes(p string, mtime time.Time) error { return os.Chtimes(p, mtime, mtime) } + +func (LocalFS) ValidName(name string) bool { + invalid := `/` + if runtime.GOOS == "windows" { + // a colon names an alternate data stream + invalid = `/\:` + } + return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, invalid) +} + +func (LocalFS) CreateNew(p string) (io.WriteCloser, error) { + return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) +} + +func (LocalFS) Rename(oldPath, newPath string) error { + err := os.Rename(oldPath, newPath) + if err != nil && runtime.GOOS == "windows" { + // Windows can't replace a read-only file + if info, statErr := os.Lstat(newPath); statErr == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o200 == 0 { + if os.Chmod(newPath, 0o600) == nil { + return os.Rename(oldPath, newPath) + } + } + } + return err +} + +func (LocalFS) Symlink(target, p string) error { + err := os.Symlink(target, p) + // ERROR_PRIVILEGE_NOT_HELD + if err != nil && runtime.GOOS == "windows" && errors.Is(err, syscall.Errno(1314)) { + return errors.Join(ErrSymlinkNotPermitted, err) + } + return err +} + +func (LocalFS) Stat(p string) (fs.FileInfo, error) { + info, err := os.Stat(p) + return localInfo(info), err +} + +func (LocalFS) Lstat(p string) (fs.FileInfo, error) { + info, err := os.Lstat(p) + return localInfo(info), err +} + +func (LocalFS) ReadDir(p string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(p) + if err != nil { + return nil, err + } + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return nil, err + } + infos = append(infos, localInfo(info)) + } + return infos, nil +} + +// localInfo drops the group and world write bits on Windows, which reports every writable file as 0666 and every +// directory as 0777, so that files copied into a container aren't writable by everyone. +func localInfo(info fs.FileInfo) fs.FileInfo { + if info == nil || runtime.GOOS != "windows" { + return info + } + return windowsInfo{info} +} + +type windowsInfo struct{ fs.FileInfo } + +func (i windowsInfo) Mode() fs.FileMode { return i.FileInfo.Mode() &^ 0o022 } + +// RemoteError is an error from an SFTPFS, so that it can be told apart from a local one. +type RemoteError struct { + Err error +} + +func (e *RemoteError) Error() string { return e.Err.Error() } +func (e *RemoteError) Unwrap() error { return e.Err } + +func remote(err error) error { + if err == nil { + return nil + } + return &RemoteError{Err: err} +} + +// SFTPFS is a file system reached over SFTP, such as a dev container's. +type SFTPFS struct { + Client *sftp.Client + // WorkDir is what relative paths are relative to. + WorkDir string +} + +var _ FS = SFTPFS{} + +func (s SFTPFS) Abs(p string) (string, error) { + if path.IsAbs(p) { + return path.Clean(p), nil + } + return path.Join(s.WorkDir, p), nil +} + +func (SFTPFS) Join(elem ...string) string { return path.Join(elem...) } +func (SFTPFS) Base(p string) string { return path.Base(p) } +func (SFTPFS) Dir(p string) string { return path.Dir(p) } +func (SFTPFS) IsSeparator(c byte) bool { return c == '/' } +func (SFTPFS) ToSlash(p string) string { return p } +func (SFTPFS) FromSlash(p string) string { return p } + +func (SFTPFS) ValidName(name string) bool { + return name != "" && name != "." && name != ".." && !strings.Contains(name, "/") +} + +func (s SFTPFS) Stat(p string) (fs.FileInfo, error) { + info, err := s.Client.Stat(p) + return info, remote(err) +} + +func (s SFTPFS) Lstat(p string) (fs.FileInfo, error) { + info, err := s.Client.Lstat(p) + return info, remote(err) +} + +func (s SFTPFS) ReadDir(p string) ([]fs.FileInfo, error) { + infos, err := s.Client.ReadDir(p) + return infos, remote(err) +} + +func (s SFTPFS) Readlink(p string) (string, error) { + target, err := s.Client.ReadLink(p) + return target, remote(err) +} + +func (s SFTPFS) Open(p string) (io.ReadCloser, error) { + f, err := s.Client.Open(p) + if err != nil { + return nil, remote(err) + } + return f, nil +} + +func (s SFTPFS) CreateNew(p string) (io.WriteCloser, error) { + f, err := s.Client.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if err != nil { + return nil, remote(err) + } + // the server creates files as 0644, so restrict it before writing anything + err = f.Chmod(0o600) + if err != nil { + _ = f.Close() + _ = s.Client.Remove(p) + return nil, remote(err) + } + return f, nil +} + +func (s SFTPFS) Rename(oldPath, newPath string) error { + return remote(s.Client.PosixRename(oldPath, newPath)) +} + +func (s SFTPFS) Mkdir(p string) error { return remote(s.Client.Mkdir(p)) } +func (s SFTPFS) MkdirAll(p string) error { return remote(s.Client.MkdirAll(p)) } +func (s SFTPFS) Remove(p string) error { return remote(s.Client.Remove(p)) } +func (s SFTPFS) Symlink(target, p string) error { return remote(s.Client.Symlink(target, p)) } +func (s SFTPFS) Chmod(p string, m fs.FileMode) error { + return remote(s.Client.Chmod(p, m)) +} +func (s SFTPFS) Chtimes(p string, mtime time.Time) error { + return remote(s.Client.Chtimes(p, mtime, mtime)) +} diff --git a/pkg/ssh/config.go b/pkg/ssh/config.go index ead3f8c9f..c2dbe3445 100644 --- a/pkg/ssh/config.go +++ b/pkg/ssh/config.go @@ -163,6 +163,38 @@ func GetUser(workspaceID string, sshConfigPath string) (string, error) { return user, nil } +// GetWorkdir returns the folder DevPod's SSH config entry for the workspace opens, which `devpod up` sets to the +// workspace folder in the container. It returns "" if the entry doesn't set one. +func GetWorkdir(workspaceID string, sshConfigPath string) (string, error) { + path, err := ResolveSSHConfigPath(sshConfigPath) + if err != nil { + return "", errors.Wrap(err, "Invalid ssh config path") + } + + workdir := "" + _, err = transformHostSection(path, workspaceID+"."+"devpod", func(line string) string { + if strings.HasPrefix(strings.TrimSpace(line), "ProxyCommand ") { + workdir = parseWorkdirFlag(line) + } + return line + }) + if err != nil { + return "", err + } + + return workdir, nil +} + +// parseWorkdirFlag returns the value of the `--workdir ""` that addHostSection adds to the ProxyCommand. +func parseWorkdirFlag(proxyCommand string) string { + _, value, found := strings.Cut(proxyCommand, ` --workdir "`) + if !found { + return "" + } + value, _, _ = strings.Cut(value, `"`) + return value +} + func RemoveFromConfig(workspaceID string, sshConfigPath string, log log.Logger) error { configLock.Lock() defer configLock.Unlock() diff --git a/pkg/ssh/config_test.go b/pkg/ssh/config_test.go index ced975b9d..d4b32feb8 100644 --- a/pkg/ssh/config_test.go +++ b/pkg/ssh/config_test.go @@ -322,3 +322,21 @@ Host testhost }) } } + +func TestParseWorkdirFlag(t *testing.T) { + for _, workdir := range []string{"", "/workspaces/foo", "/home/me/my project"} { + config, err := addHostSection("", "/path/to/devpod", "foo.devpod", "me", "default", "foo", workdir, "", false, true, "") + if err != nil { + t.Fatal(err) + } + got := "" + for _, line := range strings.Split(config, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "ProxyCommand ") { + got = parseWorkdirFlag(line) + } + } + if got != workdir { + t.Errorf("expected workdir %q, got %q from:\n%s", workdir, got, config) + } + } +} diff --git a/pkg/ssh/server/sftp_handler.go b/pkg/ssh/server/sftp_handler.go index 514552832..4a32bc326 100644 --- a/pkg/ssh/server/sftp_handler.go +++ b/pkg/ssh/server/sftp_handler.go @@ -1,49 +1,54 @@ package server import ( - "errors" - "io" + "os" + "os/exec" "github.com/curityio/devpod/pkg/command" "github.com/curityio/devpod/pkg/log" "github.com/curityio/devpod/pkg/ssh/sshd" - "github.com/pkg/sftp" "github.com/sirupsen/logrus" ) +// sftpHandler serves an SFTP session from a `helper sftp-server` process started in the user's home directory, the +// way OpenSSH runs sftp-server, so that relative paths are relative to the home directory. It doesn't serve SFTP +// in-process with sftp.WithServerWorkingDirectory, because that also makes relative symlink targets absolute. func sftpHandler(sess sshd.Session, currentUser string, log log.Logger) { - writer := log.Writer(logrus.DebugLevel, false) - defer writer.Close() - user := sess.User() if user == currentUser { user = "" } - workingDir, _ := command.GetHome(user) - serverOptions := []sftp.ServerOption{ - sftp.WithDebug(writer), - sftp.WithServerWorkingDirectory(workingDir), + workingDir, err := command.GetHome(user) + if err != nil { + log.Debugf("sftp server: find home directory of %q: %v", user, err) } - server, err := sftp.NewServer( - sess, - serverOptions..., - ) + + executable, err := os.Executable() if err != nil { - log.Debugf("sftp server init error: %s\n", err) + log.Debugf("sftp server: find executable: %v", err) + _ = sess.Exit(1) return } - defer server.Close() - // serve - err = server.Serve() - if errors.Is(err, io.EOF) { - _ = sess.Exit(0) - return + args := []string{"helper", "sftp-server"} + if log.GetLevel() == logrus.DebugLevel { + args = append(args, "--debug") } + writer := log.Writer(logrus.DebugLevel, false) + defer writer.Close() + + cmd := exec.Command(executable, args...) + cmd.Dir = workingDir + cmd.Stdin = sess + cmd.Stdout = sess + cmd.Stderr = writer + err = cmd.Run() if err != nil { log.Debugf("sftp server completed with error: %v", err) + _ = sess.Exit(1) + return } - _ = sess.Exit(1) + _ = sess.Exit(0) }