Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion docs/CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ caching, gating, and output:
order; `exitCode` is the first failing command's (0 when all succeed);
`durationMs` covers the whole sequence
- `cache` fingerprints every command in the list — changing any entry busts
the cache
the cache (see [Per-command overrides](#per-command-overrides) for how
`env`/`dir`/`silent` participate)
- `require`/`skip-if` gate the whole sequence, and templating expands each
entry's command/params individually
- `retries:` replays the **whole sequence** from the first command on each
Expand All @@ -176,6 +177,58 @@ execution path. Setting both `command` and `commands` on one group is a
load error, as is a string-form entry on a group with no `shell:`, or an
empty `commands:` list.

### Per-command overrides

Map-form entries accept three optional keys. String-form entries do not: the
shape selects the capabilities, so an entry needing an override is written as a
map.

| Key | Type | Effect |
| -------- | ------ | ----------------------------------------------------------------------------- |
| `env` | map | Layers over the group's `env:` for this command only. |
| `dir` | string | Runs this command in another directory, without needing a shell. |
| `silent` | bool | Suppresses live streaming; the output is still captured. |

```yaml
groups:
- name: release
env:
BUILD_MODE: release
commands:
- { command: go, params: [build, "./..."], env: { CGO_ENABLED: "0" } }
- { command: ./package.sh, dir: dist }
- { command: ./notify.sh, silent: true }
```

`env` precedence, lowest to highest: process environment, global `env:`, group
`env:`, command `env:`. A command's `env` applies to that command alone and does
not leak into the entries around it.

`dir` is resolved relative to the process working directory and must not be
empty. It is the reason to prefer a map entry over `cd x && ./y`: the latter is
a string-form entry, which forces `shell:` on the whole group and gives up safe
argv execution for every command in it.

`silent` affects only what is streamed to your terminal. Capture is unchanged,
so `{{ output }}` references, the JSON event stream, and cache replay all still
see the full output.

Both `env` values and `dir` are templated like `command` and `params`:

```yaml
commands:
- command: ./package.sh
dir: 'dist/{{ env "TARGET" }}'
env:
SHA: '{{ output "build" }}'
```

`env` and `dir` fold into the cache fingerprint, so changing either re-runs the
group. `silent` does not: it changes nothing about what the command does or
produces, so toggling it keeps a valid cached result. A group that uses none of
these keys keeps the fingerprint it had before they existed — upgrading does not
invalidate existing caches.

### Referencing other groups' output

A group's `command` or any of its `params` can include
Expand Down
14 changes: 12 additions & 2 deletions internal/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
"encoding/hex"
"fmt"
"io"
"maps"
"os"
"slices"
"sort"
"time"

Expand Down Expand Up @@ -35,8 +37,8 @@ type Store interface {
}

// Compute returns a content fingerprint for the given cache spec. The
// fingerprint changes when the method, any command/param/form in the group's
// command list, or any matched input file changes. For shell-form entries the
// fingerprint changes when the method, any command/param/form/env/dir in the
// group's command list, or any matched input file changes. For shell-form entries the
// fingerprint also changes when the shell program changes. A glob that matches
// nothing contributes nothing, so adding the first matching file naturally
// changes the fingerprint.
Expand All @@ -57,6 +59,14 @@ func Compute(spec *config.Cache, shell string, commands []config.CommandSpec) (s
for _, p := range c.Params {
fmt.Fprintf(h, "%s\x01", p)
}
// Emitted only when set: a config using neither must keep hashing as it
// did before they existed, or every cache entry silently invalidates.
if c.Dir != "" {
fmt.Fprintf(h, "%s\x03", c.Dir)
}
for _, k := range slices.Sorted(maps.Keys(c.Env)) {
fmt.Fprintf(h, "%s\x04%s\x03", k, c.Env[k])
}
fmt.Fprintf(h, "\x02")
}

Expand Down
136 changes: 136 additions & 0 deletions internal/cache/percommand_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package cache

import (
"crypto/sha256"
"encoding/hex"
"fmt"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/quike/keepup/internal/config"
)

// fpFor computes a fingerprint for one command against a fixed input file.
func fpFor(t *testing.T, dir string, cs config.CommandSpec) string {
t.Helper()
spec := &config.Cache{Method: config.CacheHash, Reads: []string{filepath.Join(dir, "*.go")}}
fp, err := Compute(spec, "", []config.CommandSpec{cs})
require.NoError(t, err)
return fp
}

func TestCompute_PerCommandKnobs(t *testing.T) {
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "a.go"), "package main\n")
base := config.CommandSpec{Command: "go", Params: []string{"build"}}

tests := []struct {
name string
spec config.CommandSpec
wantBust bool
}{
{
name: "env value change busts",
spec: config.CommandSpec{Command: "go", Params: []string{"build"}, Env: map[string]string{"CGO_ENABLED": "0"}},
wantBust: true,
},
{
name: "dir change busts",
spec: config.CommandSpec{Command: "go", Params: []string{"build"}, Dir: "sub"},
wantBust: true,
},
{
name: "silent is display-only and must NOT bust",
spec: config.CommandSpec{Command: "go", Params: []string{"build"}, Silent: true},
wantBust: false,
},
{
name: "empty env map is the same as no env",
spec: config.CommandSpec{Command: "go", Params: []string{"build"}, Env: map[string]string{}},
wantBust: false,
},
}

baseFP := fpFor(t, dir, base)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := fpFor(t, dir, tc.spec)
if tc.wantBust {
assert.NotEqual(t, baseFP, got)
return
}
assert.Equal(t, baseFP, got)
})
}
}

// Go map iteration order is randomized, so an unsorted encoding would make the
// fingerprint differ between runs of the same config.
func TestCompute_EnvOrderIsDeterministic(t *testing.T) {
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "a.go"), "package main\n")
spec := config.CommandSpec{
Command: "go",
Env: map[string]string{"A": "1", "B": "2", "C": "3", "D": "4", "E": "5"},
}
want := fpFor(t, dir, spec)
for range 20 {
assert.Equal(t, want, fpFor(t, dir, spec))
}
}

// Two different env maps must not hash the same just because their
// concatenated bytes could line up.
func TestCompute_EnvPairsAreUnambiguous(t *testing.T) {
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "a.go"), "package main\n")
a := config.CommandSpec{Command: "go", Env: map[string]string{"A": "BC"}}
b := config.CommandSpec{Command: "go", Env: map[string]string{"AB": "C"}}
assert.NotEqual(t, fpFor(t, dir, a), fpFor(t, dir, b))
}

// Guards the upgrade path: without this, every existing cache entry invalidates.
func TestCompute_UnchangedConfigKeepsItsFingerprint(t *testing.T) {
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "a.go"), "package main\n")
spec := &config.Cache{Method: config.CacheHash, Reads: []string{filepath.Join(dir, "*.go")}}

got, err := Compute(spec, "/bin/sh", []config.CommandSpec{
{Command: "go", Params: []string{"build", "./..."}},
{Command: "echo done", IsShell: true},
})
require.NoError(t, err)

assert.Equal(t, legacyV3(t, spec, "/bin/sh", []config.CommandSpec{
{Command: "go", Params: []string{"build", "./..."}},
{Command: "echo done", IsShell: true},
}), got)
}

// legacyV3 reproduces the byte stream Compute produced before env/dir existed.
// Written out independently so the back-compat assertion cannot drift along
// with the implementation it is guarding.
func legacyV3(t *testing.T, spec *config.Cache, shell string, commands []config.CommandSpec) string {
t.Helper()
h := sha256.New()
fmt.Fprintf(h, "v3\x00%s\x00", spec.Method)
for _, c := range commands {
fmt.Fprintf(h, "%s\x00%t\x00", c.Command, c.IsShell)
if c.IsShell {
fmt.Fprintf(h, "%s\x00", shell)
}
for _, p := range c.Params {
fmt.Fprintf(h, "%s\x01", p)
}
fmt.Fprintf(h, "\x02")
}
files, err := resolveGlobs(spec.Reads)
require.NoError(t, err)
for _, f := range files {
require.NoError(t, hashFile(h, spec.Method, f))
}
return "sha256:" + hex.EncodeToString(h.Sum(nil))
}
Loading