From c540b45ee291ec3584e36e169c7c25b38d2adf54 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:17 +0200 Subject: [PATCH 1/6] postgresql: migrate to PostgreSQL 16 compatibility - Replace wal_keep_segments with wal_keep_size=2048 (removed in PG13) - Replace wal_level=hot_standby with wal_level=replica - Set password_encryption=md5 (PG16 defaults to scram-sha-256) - Replace recovery.conf with standby.signal and config-based recovery settings - Use pg_ctl promote instead of promote_trigger_file (removed in PG16) - Grant ALL ON SCHEMA public TO PUBLIC in template1 (PG15+ restriction) - Use CREATE DATABASE ... OWNER in postgres API --- .../postgresql/cmd/flynn-postgres-api/main.go | 4 +- appliance/postgresql/process.go | 76 +++++++++++-------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/appliance/postgresql/cmd/flynn-postgres-api/main.go b/appliance/postgresql/cmd/flynn-postgres-api/main.go index f9f6fa074..a381fc3c0 100755 --- a/appliance/postgresql/cmd/flynn-postgres-api/main.go +++ b/appliance/postgresql/cmd/flynn-postgres-api/main.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "context" "github.com/flynn/flynn/discoverd/client" "github.com/flynn/flynn/pkg/httphelper" "github.com/flynn/flynn/pkg/postgres" @@ -13,7 +14,6 @@ import ( "github.com/flynn/flynn/pkg/resource" "github.com/flynn/flynn/pkg/shutdown" "github.com/julienschmidt/httprouter" - "context" ) const ( @@ -78,7 +78,7 @@ func (p *pgAPI) createDatabase(ctx context.Context, w http.ResponseWriter, req * httphelper.Error(w, err) return } - if err := p.db.Exec(fmt.Sprintf(`CREATE DATABASE "%s"`, database)); err != nil { + if err := p.db.Exec(fmt.Sprintf(`CREATE DATABASE "%s" OWNER "%s"`, database, username)); err != nil { p.db.Exec(fmt.Sprintf(`DROP USER "%s"`, username)) httphelper.Error(w, err) return diff --git a/appliance/postgresql/process.go b/appliance/postgresql/process.go index 4703f4196..1753b97ca 100755 --- a/appliance/postgresql/process.go +++ b/appliance/postgresql/process.go @@ -347,8 +347,9 @@ func (p *Process) assumePrimary(downstream *discoverd.Instance) (err error) { if p.running() && p.config().Role == state.RoleSync { log.Info("promoting to primary") - if err := os.WriteFile(p.triggerPath(), nil, 0655); err != nil { - log.Error("error creating trigger file", "path", p.triggerPath(), "err", err) + // PG 16 removed promote_trigger_file; use pg_ctl promote instead + if err := p.runCmd(exec.Command(p.binPath("pg_ctl"), "promote", "-D", p.dataDir)); err != nil { + log.Error("error promoting standby", "err", err) return err } @@ -369,6 +370,10 @@ func (p *Process) assumePrimary(downstream *discoverd.Instance) (err error) { if err := os.Remove(p.recoveryConfPath()); err != nil && !os.IsNotExist(err) { log.Error("error removing recovery.conf", "path", p.recoveryConfPath(), "err", err) + // non-fatal, PG 16 doesn't use recovery.conf + } + if err := os.Remove(p.standbySignalPath()); err != nil && !os.IsNotExist(err) { + log.Error("error removing standby.signal", "path", p.standbySignalPath(), "err", err) return err } @@ -483,6 +488,14 @@ func (p *Process) installExtensionsInTemplate() error { return fmt.Errorf("creating extension %s in template1: %s", ext, err) } } + + // In PostgreSQL 15+, the default CREATE privilege on the public schema was + // revoked for non-owner roles. Restore the PG14 behavior so that application + // users can create tables in the public schema of their databases. + if _, err := templateDB.Exec("GRANT ALL ON SCHEMA public TO PUBLIC"); err != nil { + return fmt.Errorf("granting public schema privileges in template1: %s", err) + } + return nil } @@ -536,12 +549,8 @@ func (p *Process) assumeStandby(upstream, downstream *discoverd.Instance) error os.Remove(p.triggerPath()) } - if err := p.writeConfig(configData{ReadOnly: true}); err != nil { - log.Error("error writing postgres.conf", "path", p.configPath(), "err", err) - return err - } - if err := p.writeRecoveryConf(upstream); err != nil { - log.Error("error writing recovery.conf", "path", p.recoveryConfPath(), "err", err) + if err := p.writeRecoveryConf(upstream, configData{ReadOnly: true}); err != nil { + log.Error("error writing recovery config", "path", p.configPath(), "err", err) return err } @@ -897,21 +906,21 @@ func (p *Process) writeConfig(d configData) error { return configTemplate.Execute(f, d) } -func (p *Process) writeRecoveryConf(upstream *discoverd.Instance) error { - data := recoveryData{ - TriggerFile: p.triggerPath(), - PrimaryInfo: fmt.Sprintf( - "host=%s port=%s user=flynn password=%s application_name=%s", - upstream.Host(), upstream.Port(), p.password, p.id, - ), - } +func (p *Process) recoveryConfigData(upstream *discoverd.Instance, d configData) configData { + d.PrimaryConnInfo = fmt.Sprintf( + "host=%s port=%s user=flynn password=%s application_name=%s", + upstream.Host(), upstream.Port(), p.password, p.id, + ) + return d +} - f, err := os.Create(p.recoveryConfPath()) - if err != nil { +func (p *Process) writeRecoveryConf(upstream *discoverd.Instance, d configData) error { + // PG 12+ uses postgresql.conf for recovery settings + standby.signal file + if err := p.writeConfig(p.recoveryConfigData(upstream, d)); err != nil { return err } - defer f.Close() - return recoveryConfTemplate.Execute(f, data) + // Create standby.signal to indicate this is a standby + return os.WriteFile(p.standbySignalPath(), nil, 0644) } func (p *Process) writeHBAConf() error { @@ -923,9 +932,14 @@ func (p *Process) configPath() string { } func (p *Process) recoveryConfPath() string { + // PG 16 no longer uses recovery.conf; keep this for cleanup of old files return p.dataPath("recovery.conf") } +func (p *Process) standbySignalPath() string { + return p.dataPath("standby.signal") +} + func (p *Process) hbaConfPath() string { return p.dataPath("pg_hba.conf") } @@ -951,6 +965,9 @@ type configData struct { TimescaleDB bool ExtWhitelist bool SHMType string + + // Recovery settings (PG 12+ uses postgresql.conf instead of recovery.conf) + PrimaryConnInfo string } var configTemplate = template.Must(template.New("postgresql.conf").Parse(` @@ -960,10 +977,10 @@ port = {{.Port}} ssl = off max_connections = 400 shared_buffers = 32MB -wal_level = hot_standby +wal_level = replica fsync = on max_wal_senders = 15 -wal_keep_segments = 128 +wal_keep_size = 2048 synchronous_commit = remote_write synchronous_standby_names = '{{.Sync}}' {{if .ReadOnly}} @@ -985,6 +1002,7 @@ datestyle = 'iso, mdy' timezone = 'UTC' client_encoding = 'UTF8' default_text_search_config = 'pg_catalog.english' +password_encryption = md5 {{if .TimescaleDB}} shared_preload_libraries = 'timescaledb' @@ -998,18 +1016,12 @@ dynamic_shared_memory_type = '{{.SHMType}}' local_preload_libraries = 'pgextwlist' extwlist.extensions = 'btree_gin,btree_gist,chkpass,citext,cube,dblink,dict_int,earthdistance,fuzzystrmatch,hstore,intarray,isn,ltree,pg_prewarm,pg_stat_statements,pg_trgm,pgcrypto,pgrouting,pgrowlocks,pgstattuple,plpgsql,plv8,postgis,postgis_topology,postgres_fdw,tablefunc,timescaledb,unaccent,uuid-ossp' {{end}} -`[1:])) - -type recoveryData struct { - PrimaryInfo string - TriggerFile string -} -var recoveryConfTemplate = template.Must(template.New("recovery.conf").Parse(` -standby_mode = on -primary_conninfo = '{{.PrimaryInfo}}' -trigger_file = '{{.TriggerFile}}' +{{if .PrimaryConnInfo}} +# Recovery settings (managed by flynn-postgres, PG 12+) +primary_conninfo = '{{.PrimaryConnInfo}}' recovery_target_timeline = 'latest' +{{end}} `[1:])) var hbaConf = []byte(` From 2ec11480cc3968307b9002e0158b1ca3f2f2327a Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:22 +0200 Subject: [PATCH 2/6] controller: fix PG16 trigger functions and schema loading - Replace RETURNS OPAQUE with RETURNS trigger in 4 migration functions (OPAQUE type removed in PG16) - Skip macOS resource fork files (._*) in JSON schema loader to prevent parse errors in containers --- controller/data/schema.go | 8 ++++---- controller/schema/schema.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/controller/data/schema.go b/controller/data/schema.go index 49a4645b2..688dafc97 100755 --- a/controller/data/schema.go +++ b/controller/data/schema.go @@ -142,7 +142,7 @@ $$ LANGUAGE plpgsql`, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now() )`, - `CREATE FUNCTION check_job_state() RETURNS OPAQUE AS $$ + `CREATE FUNCTION check_job_state() RETURNS trigger AS $$ BEGIN IF NEW.state < OLD.state THEN RAISE EXCEPTION 'invalid job state transition: % -> %', OLD.state, NEW.state USING ERRCODE = 'check_violation'; @@ -309,7 +309,7 @@ $$ LANGUAGE plpgsql`, // add a check to ensure releases only have a single "docker" // artifact, and that artifact is added first - `CREATE FUNCTION check_release_artifacts() RETURNS OPAQUE AS $$ + `CREATE FUNCTION check_release_artifacts() RETURNS trigger AS $$ BEGIN IF ( SELECT COUNT(*) @@ -359,7 +359,7 @@ $$ LANGUAGE plpgsql`, `INSERT INTO event_types (name) VALUES ('release_deletion')`, // add a trigger to prevent current app releases from being deleted - `CREATE FUNCTION check_release_delete() RETURNS OPAQUE AS $$ + `CREATE FUNCTION check_release_delete() RETURNS trigger AS $$ BEGIN IF NEW.deleted_at IS NOT NULL AND (SELECT COUNT(*) FROM apps WHERE release_id = NEW.release_id) != 0 THEN RAISE EXCEPTION 'cannot delete current app release' USING ERRCODE = 'check_violation'; @@ -432,7 +432,7 @@ $$ LANGUAGE plpgsql`, `ALTER TABLE artifacts ADD COLUMN hashes jsonb`, `ALTER TABLE artifacts ADD COLUMN size integer`, `ALTER TABLE artifacts ADD COLUMN layer_url_template text`, - `CREATE FUNCTION check_artifact_manifest() RETURNS OPAQUE AS $$ + `CREATE FUNCTION check_artifact_manifest() RETURNS trigger AS $$ BEGIN IF NEW.type = 'flynn' AND NEW.manifest IS NULL THEN RAISE EXCEPTION 'flynn artifacts must have a manifest' USING ERRCODE = 'check_violation'; diff --git a/controller/schema/schema.go b/controller/schema/schema.go index a3663fad7..d1475b73a 100755 --- a/controller/schema/schema.go +++ b/controller/schema/schema.go @@ -23,7 +23,7 @@ func Load(schemaRoot string) error { var schemaPaths []string walkFn := func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && filepath.Ext(path) == ".json" { + if !info.IsDir() && filepath.Ext(path) == ".json" && !strings.HasPrefix(filepath.Base(path), "._") { schemaPaths = append(schemaPaths, path) } return nil From aec8805185fbcb432dbec2ccc9a34bcfeca243bd Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:28 +0200 Subject: [PATCH 3/6] flannel: fix VXLAN MAC address reset after LinkSetUp - Store desired MAC in vxlanDevice struct - Re-apply deterministic MAC after Configure()->LinkSetUp() which resets the hardware address, causing VXLAN routing failures --- flannel/backend/vxlan/device.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/flannel/backend/vxlan/device.go b/flannel/backend/vxlan/device.go index 808c2eb69..06af0062c 100755 --- a/flannel/backend/vxlan/device.go +++ b/flannel/backend/vxlan/device.go @@ -20,7 +20,8 @@ type vxlanDeviceAttrs struct { } type vxlanDevice struct { - link *netlink.Vxlan + link *netlink.Vxlan + desiredMAC net.HardwareAddr } func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) { @@ -79,7 +80,8 @@ func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) { } return &vxlanDevice{ - link: link, + link: link, + desiredMAC: link.HardwareAddr, }, nil } @@ -134,6 +136,20 @@ func (dev *vxlanDevice) Configure(ipn ip.IP4Net) error { return fmt.Errorf("failed to set interface %s to UP state: %s", dev.link.Attrs().Name, err) } + // Re-apply the desired MAC address after LinkSetUp, because some kernels + // regenerate the VXLAN MAC when the interface transitions to UP state. + if dev.desiredMAC != nil { + current, _ := netlink.LinkByIndex(dev.link.Index) + if current != nil && !macEqual(current.Attrs().HardwareAddr, dev.desiredMAC) { + log.Infof("MAC changed after LinkSetUp (got %s, want %s), re-applying", current.Attrs().HardwareAddr, dev.desiredMAC) + if err := netlink.LinkSetHardwareAddr(dev.link, dev.desiredMAC); err != nil { + log.Warningf("failed to re-apply MAC on %s: %v", dev.link.Name, err) + } else { + dev.link.HardwareAddr = dev.desiredMAC + } + } + } + // explicitly add a route since there might be a route for a subnet already // installed by Docker and then it won't get auto added route := netlink.Route{ From 4b4de5f4c21b72487592ed0ef3783b81aed32ed3 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:33 +0200 Subject: [PATCH 4/6] host: call TUF client Update() after initialization - Without Update(), the TUF client has stale metadata and fails to verify downloaded layer targets --- host/libcontainer_backend.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/host/libcontainer_backend.go b/host/libcontainer_backend.go index 505175a50..c1816af7a 100755 --- a/host/libcontainer_backend.go +++ b/host/libcontainer_backend.go @@ -39,6 +39,7 @@ import ( "github.com/flynn/flynn/pkg/shutdown" "github.com/flynn/flynn/pkg/syslog/rfc5424" "github.com/flynn/flynn/pkg/term" + "github.com/flynn/flynn/pkg/tufconfig" "github.com/flynn/flynn/pkg/tufutil" "github.com/flynn/flynn/pkg/verify" tuf "github.com/flynn/go-tuf/client" @@ -117,6 +118,21 @@ func NewLibcontainerBackend(config *LibcontainerConfig) (Backend, error) { if err != nil { return nil, fmt.Errorf("error initializing TUF client: %s", err) } + // Update the local TUF metadata from the remote repository + // so that target lookups in Download() can find the targets. + // Initialize root keys if this is a fresh local store. + if _, err := tufClient.Update(); err != nil { + if err == tuf.ErrNoRootKeys { + if err := tufClient.Init(tufconfig.RootKeys, 1); err != nil { + return nil, fmt.Errorf("error initializing TUF root keys: %s", err) + } + if _, err := tufClient.Update(); err != nil && !tuf.IsLatestSnapshot(err) { + return nil, fmt.Errorf("error updating TUF metadata after init: %s", err) + } + } else if !tuf.IsLatestSnapshot(err) { + return nil, fmt.Errorf("error updating TUF metadata: %s", err) + } + } l.tufClient = tufClient } l.httpClient = &http.Client{Transport: &http.Transport{ From 06bf70c7e8ccfa579c2a1a1ca18a6259ee3496d9 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:38 +0200 Subject: [PATCH 5/6] migrate: propagate CREATE TABLE errors instead of silently ignoring - schema_migrations CREATE TABLE failure was swallowed, causing nil pointer panics later when the table didn't exist --- pkg/postgres/migrate.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/postgres/migrate.go b/pkg/postgres/migrate.go index bbf0ea1f0..59fcfcbd6 100755 --- a/pkg/postgres/migrate.go +++ b/pkg/postgres/migrate.go @@ -4,8 +4,8 @@ import ( "strconv" "time" - "github.com/jackc/pgx" "github.com/inconshreveable/log15" + "github.com/jackc/pgx" ) type Step func(*DBTx) error @@ -39,7 +39,9 @@ func (m Migrations) Migrate(db *DB) error { var initialized bool for _, migration := range m { if !initialized { - db.Exec("CREATE TABLE IF NOT EXISTS schema_migrations (id bigint PRIMARY KEY)") + if err := db.Exec("CREATE TABLE IF NOT EXISTS schema_migrations (id bigint PRIMARY KEY)"); err != nil { + return err + } initialized = true } From fbf2929927150c75263bb9f81093d26bdb2f664b Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2026 22:24:47 +0200 Subject: [PATCH 6/6] export-tuf: add ExtraDirs, package layers, and Ubuntu Noble support - Add ExtraDirs map to imageSpec for copying directories into layers (used for controller JSON schema files) - Add --package-layer-dir and PackageScript for pre-built package layers - Add --skip-base-layers to skip rebuilding base OS layers - Remap /bin/ to /usr/bin/ for Ubuntu Noble (merged /bin symlink) - Use GitHub Releases URLs for layer downloads instead of GitHub Pages - Filter macOS resource fork files (._*) from directory copies - Add isUbuntuNobleBase() detection for Noble cloud images --- script/export-tuf/main.go | 411 +++++++++++++++++++++++++++++++++----- 1 file changed, 362 insertions(+), 49 deletions(-) diff --git a/script/export-tuf/main.go b/script/export-tuf/main.go index 34a0255db..aa680f7cc 100755 --- a/script/export-tuf/main.go +++ b/script/export-tuf/main.go @@ -38,11 +38,13 @@ import ( // imageSpec defines a Flynn component image: what base layers it inherits, // what binaries and files it contains, and its entrypoint. type imageSpec struct { - Name string - Base string // base image name for layer inheritance - Binaries map[string]string // source binary name -> dest path in squashfs - ExtraFiles map[string]string // source file (relative to source-dir) -> dest path - Entrypoint *ct.ImageEntrypoint + Name string + Base string // base image name for layer inheritance + Binaries map[string]string // source binary name -> dest path in squashfs + ExtraFiles map[string]string // source file (relative to source-dir) -> dest path + ExtraDirs map[string]string // source dir (relative to source-dir) -> dest path + Entrypoint *ct.ImageEntrypoint + PackageScript string // path relative to source-dir for package install script (run in chroot on base layer) } func main() { @@ -54,12 +56,14 @@ func main() { func run() error { var ( - tufDir string - buildDir string - sourceDir string - version string - layerCache string - tufRepo string + tufDir string + buildDir string + sourceDir string + version string + layerCache string + tufRepo string + skipBase bool + pkgLayerDir string ) // Parse flags manually (avoid adding dependencies) @@ -76,6 +80,10 @@ func run() error { layerCache = strings.TrimPrefix(arg, "--layer-cache=") } else if strings.HasPrefix(arg, "--tuf-repo=") { tufRepo = strings.TrimPrefix(arg, "--tuf-repo=") + } else if arg == "--skip-base-layers" { + skipBase = true + } else if strings.HasPrefix(arg, "--package-layer-dir=") { + pkgLayerDir = strings.TrimPrefix(arg, "--package-layer-dir=") } else if arg == "--help" || arg == "-h" { printUsage() return nil @@ -100,16 +108,19 @@ func run() error { } e := &exporter{ - tufDir: tufDir, - buildDir: buildDir, - binDir: binDir, - sourceDir: sourceDir, - version: version, - layerCache: layerCache, - tufRepoURL: tufRepo, - layerURLTpl: fmt.Sprintf("%s?target=/layers/{id}.squashfs", tufRepo), - baseLayers: make(map[string][]*ct.ImageLayer), - artifacts: make(map[string]*ct.Artifact), + tufDir: tufDir, + buildDir: buildDir, + binDir: binDir, + sourceDir: sourceDir, + version: version, + layerCache: layerCache, + tufRepoURL: tufRepo, + layerURLTpl: fmt.Sprintf("https://github.com/consolving/flynn-tuf-repo/releases/download/%s/{id}.squashfs", version), + skipBaseLayers: skipBase, + pkgLayerDir: pkgLayerDir, + baseLayers: make(map[string][]*ct.ImageLayer), + packageLayers: make(map[string]*ct.ImageLayer), + artifacts: make(map[string]*ct.Artifact), } return e.Run() @@ -125,21 +136,26 @@ Options: --version=VERSION Version string (e.g., v20250412.0) --layer-cache=DIR Path to layer cache directory --tuf-repo=URL TUF repository URL [default: https://consolving.github.io/flynn-tuf-repo/repository] + --skip-base-layers Use cached base layers instead of building them + --package-layer-dir=DIR Directory with pre-built {name}-packages.squashfs files `) } type exporter struct { - tufDir string - buildDir string - binDir string - sourceDir string - version string - layerCache string - tufRepoURL string - layerURLTpl string - - baseLayers map[string][]*ct.ImageLayer // base image name -> accumulated layers - artifacts map[string]*ct.Artifact // image name -> artifact + tufDir string + buildDir string + binDir string + sourceDir string + version string + layerCache string + tufRepoURL string + layerURLTpl string + skipBaseLayers bool + pkgLayerDir string + + baseLayers map[string][]*ct.ImageLayer // base image name -> accumulated layers + packageLayers map[string]*ct.ImageLayer // package script path -> cached layer + artifacts map[string]*ct.Artifact // image name -> artifact } func (e *exporter) Run() error { @@ -184,37 +200,138 @@ func (e *exporter) Run() error { func (e *exporter) buildBaseLayers() error { // Build the base OS layers in dependency order. - // Each base layer becomes a squashfs file in the layer cache. - // - // Dependency tree: - // busybox (standalone) - // ubuntu-noble (standalone) - // ubuntu-noble (standalone, needed only for host image) - bases := []struct { name string script string }{ {"busybox", "builder/img/busybox.sh"}, {"ubuntu-noble", "builder/img/ubuntu-noble.sh"}, - // ubuntu-noble needed for host image but host image also needs - // kernel packages which require a full apt - skip for now as - // the host image will use ubuntu-noble in the simplified pipeline } for _, base := range bases { - fmt.Printf(" Building base layer: %s\n", base.name) - layer, err := e.buildBaseLayer(base.name, base.script) - if err != nil { - return fmt.Errorf("building %s: %s", base.name, err) + if e.skipBaseLayers { + // Load from cache instead of building + fmt.Printf(" Loading base layer from cache: %s\n", base.name) + layer, err := e.loadBaseLayerFromCache(base.name) + if err != nil { + return fmt.Errorf("loading cached %s: %s (try without --skip-base-layers)", base.name, err) + } + e.baseLayers[base.name] = []*ct.ImageLayer{layer} + fmt.Printf(" -> %s: id=%s size=%d\n", base.name, layer.ID, layer.Length) + } else { + fmt.Printf(" Building base layer: %s\n", base.name) + layer, err := e.buildBaseLayer(base.name, base.script) + if err != nil { + return fmt.Errorf("building %s: %s", base.name, err) + } + e.baseLayers[base.name] = []*ct.ImageLayer{layer} + fmt.Printf(" -> %s: id=%s size=%d\n", base.name, layer.ID, layer.Length) } - e.baseLayers[base.name] = []*ct.ImageLayer{layer} - fmt.Printf(" -> %s: id=%s size=%d\n", base.name, layer.ID, layer.Length) } return nil } +// loadBaseLayerFromCache finds the cached layer for the given base name. +// It uses the known base layer hashes from the layer cache's JSON metadata. +func (e *exporter) loadBaseLayerFromCache(name string) (*ct.ImageLayer, error) { + // Scan the cache for layers and check their JSON metadata to identify base layers. + // Base layers built by buildBaseLayer are large squashfs files. + // For busybox: ~2.5 MB with /bin/busybox, /etc/passwd + // For ubuntu-noble: ~158 MB with full Ubuntu rootfs + + entries, err := os.ReadDir(e.layerCache) + if err != nil { + return nil, err + } + + var bestID string + var bestSize int64 + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".squashfs") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + id := strings.TrimSuffix(entry.Name(), ".squashfs") + size := info.Size() + + if name == "ubuntu-noble" { + // Ubuntu Noble base is ~150-170 MB, has /etc/cloud/ directory + if size > 100*1024*1024 { + sqfsPath := filepath.Join(e.layerCache, entry.Name()) + if isUbuntuNobleBase(sqfsPath) && size > bestSize { + bestSize = size + bestID = id + } + } + } else if name == "busybox" { + // Busybox base is ~2-3 MB (not 4KB which would be slugrunner) + if size > 1*1024*1024 && size < 10*1024*1024 { + // Verify it's actually busybox by checking it has /bin/busybox + sqfsPath := filepath.Join(e.layerCache, entry.Name()) + if isBusyboxLayer(sqfsPath) { + if bestSize == 0 || size > bestSize { + bestSize = size + bestID = id + } + } + } + } + } + + if bestID == "" { + return nil, fmt.Errorf("no cached layer found for %s", name) + } + + // Read the cached layer and verify hash + squashfsPath := filepath.Join(e.layerCache, bestID+".squashfs") + data, err := os.ReadFile(squashfsPath) + if err != nil { + return nil, err + } + + digest := sha512.Sum512_256(data) + computedID := hex.EncodeToString(digest[:]) + if computedID != bestID { + return nil, fmt.Errorf("cache integrity error: expected %s got %s", bestID, computedID) + } + + return &ct.ImageLayer{ + ID: bestID, + Type: ct.ImageLayerTypeSquashfs, + Length: int64(len(data)), + Hashes: map[string]string{ + "sha512_256": bestID, + }, + }, nil +} + +// isBusyboxLayer checks if a squashfs file contains /bin/busybox +func isBusyboxLayer(sqfsPath string) bool { + // Use unsquashfs to check for busybox binary + cmd := exec.Command("unsquashfs", "-l", sqfsPath) + out, err := cmd.Output() + if err != nil { + return false + } + return strings.Contains(string(out), "/bin/busybox") +} + +// isUbuntuNobleBase checks if a squashfs is an Ubuntu Noble cloud-image base layer +// (as opposed to a package-install diff layer that might also be large) +func isUbuntuNobleBase(sqfsPath string) bool { + cmd := exec.Command("unsquashfs", "-l", sqfsPath) + out, err := cmd.Output() + if err != nil { + return false + } + // Cloud images have /etc/cloud/ directory; package diff layers don't + return strings.Contains(string(out), "/etc/cloud/") +} + func (e *exporter) buildBaseLayer(name, scriptPath string) (*ct.ImageLayer, error) { scriptAbs := filepath.Join(e.sourceDir, scriptPath) @@ -312,9 +429,17 @@ func (e *exporter) buildComponentImage(spec imageSpec) error { } defer os.RemoveAll(tmpDir) + // For ubuntu-noble based images, /bin is a symlink to usr/bin. + // Component layers must place files in /usr/bin/ to avoid creating + // a real /bin/ directory that shadows the base layer's symlink in overlayfs. + remapBin := spec.Base == "ubuntu-noble" + // Copy binaries for srcName, destPath := range spec.Binaries { srcPath := filepath.Join(e.binDir, srcName) + if remapBin && strings.HasPrefix(destPath, "/bin/") { + destPath = "/usr" + destPath + } dst := filepath.Join(tmpDir, destPath) if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err @@ -327,6 +452,9 @@ func (e *exporter) buildComponentImage(spec imageSpec) error { // Copy extra files for srcRel, destPath := range spec.ExtraFiles { srcPath := filepath.Join(e.sourceDir, srcRel) + if remapBin && strings.HasPrefix(destPath, "/bin/") { + destPath = "/usr" + destPath + } dst := filepath.Join(tmpDir, destPath) if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err @@ -336,6 +464,30 @@ func (e *exporter) buildComponentImage(spec imageSpec) error { } } + // Copy extra directories + for srcRel, destPath := range spec.ExtraDirs { + srcPath := filepath.Join(e.sourceDir, srcRel) + dstBase := filepath.Join(tmpDir, destPath) + err := filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(srcPath, path) + dst := filepath.Join(dstBase, rel) + if info.IsDir() { + return os.MkdirAll(dst, 0755) + } + // Skip macOS resource fork files + if strings.HasPrefix(filepath.Base(path), "._") { + return nil + } + return copyFile(path, dst, 0644) + }) + if err != nil { + return fmt.Errorf("copying extra dir %s: %s", srcRel, err) + } + } + // Create squashfs from the directory squashfsPath := filepath.Join(tmpDir, "layer.squashfs") cmd := exec.Command("mksquashfs", tmpDir, squashfsPath, "-noappend", @@ -352,11 +504,18 @@ func (e *exporter) buildComponentImage(spec imageSpec) error { return err } - // Build the ImageManifest with base layers + component layer + // Build the ImageManifest with base layers + optional package layer + component layer var allLayers []*ct.ImageLayer if baseLayers, ok := e.baseLayers[spec.Base]; ok { allLayers = append(allLayers, baseLayers...) } + if spec.PackageScript != "" { + pkgLayer, err := e.buildPackageLayer(spec) + if err != nil { + return fmt.Errorf("building package layer for %s: %s", spec.Name, err) + } + allLayers = append(allLayers, pkgLayer) + } allLayers = append(allLayers, componentLayer) manifest := ct.ImageManifest{ @@ -393,6 +552,150 @@ func (e *exporter) buildComponentImage(spec imageSpec) error { return nil } +// buildPackageLayer runs a package installation script in a chroot on the base layer, +// producing a squashfs layer containing only the changes (installed packages, users, etc.). +// Results are cached by package script path so multiple images sharing the same script +// only build the layer once. +func (e *exporter) buildPackageLayer(spec imageSpec) (*ct.ImageLayer, error) { + // Check cache + if layer, ok := e.packageLayers[spec.PackageScript]; ok { + fmt.Printf(" Using cached package layer for %s\n", spec.PackageScript) + return layer, nil + } + + fmt.Printf(" Building package layer: %s\n", spec.PackageScript) + + // Check for pre-built package layer in pkgLayerDir + if e.pkgLayerDir != "" { + prebuiltPath := filepath.Join(e.pkgLayerDir, spec.Name+"-packages.squashfs") + if _, err := os.Stat(prebuiltPath); err == nil { + fmt.Printf(" Using pre-built package layer: %s\n", prebuiltPath) + layer, err := e.importSquashfs(prebuiltPath) + if err != nil { + return nil, fmt.Errorf("importing pre-built package layer: %s", err) + } + e.packageLayers[spec.PackageScript] = layer + fmt.Printf(" -> package layer: id=%s size=%d\n", layer.ID[:16], layer.Length) + return layer, nil + } + } + + // Find the base layer squashfs file + baseLayers := e.baseLayers[spec.Base] + if len(baseLayers) == 0 { + return nil, fmt.Errorf("no base layers for %s", spec.Base) + } + baseLayerID := baseLayers[0].ID + baseSquashfs := filepath.Join(e.layerCache, baseLayerID+".squashfs") + + // Create temp dirs for overlay mount + workDir, err := os.MkdirTemp("", "flynn-pkg-"+spec.Name) + if err != nil { + return nil, err + } + defer os.RemoveAll(workDir) + + lowerDir := filepath.Join(workDir, "lower") + upperDir := filepath.Join(workDir, "upper") + mergedDir := filepath.Join(workDir, "merged") + overlayWork := filepath.Join(workDir, "work") + for _, d := range []string{lowerDir, upperDir, mergedDir, overlayWork} { + if err := os.MkdirAll(d, 0755); err != nil { + return nil, err + } + } + + // Mount base squashfs + cmd := exec.Command("mount", "-t", "squashfs", "-o", "loop,ro", baseSquashfs, lowerDir) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("mount base squashfs: %s", err) + } + defer exec.Command("umount", "-l", lowerDir).Run() + + // Mount overlayfs + overlayOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, overlayWork) + cmd = exec.Command("mount", "-t", "overlay", "overlay", "-o", overlayOpts, mergedDir) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("mount overlay: %s", err) + } + defer exec.Command("umount", "-l", mergedDir).Run() + + // Bind-mount essential filesystems for chroot + for _, m := range []struct{ src, dst string }{ + {"/proc", filepath.Join(mergedDir, "proc")}, + {"/sys", filepath.Join(mergedDir, "sys")}, + {"/dev", filepath.Join(mergedDir, "dev")}, + } { + os.MkdirAll(m.dst, 0755) + cmd = exec.Command("mount", "--bind", m.src, m.dst) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("bind mount %s: %s", m.src, err) + } + defer exec.Command("umount", "-l", m.dst).Run() + } + + // Copy resolv.conf for network access during package install + resolvSrc := "/etc/resolv.conf" + resolvDst := filepath.Join(mergedDir, "etc/resolv.conf") + copyFile(resolvSrc, resolvDst, 0644) + + // Copy the package script into the chroot + scriptSrc := filepath.Join(e.sourceDir, spec.PackageScript) + scriptDst := filepath.Join(mergedDir, "tmp/packages.sh") + os.MkdirAll(filepath.Dir(scriptDst), 0755) + if err := copyFile(scriptSrc, scriptDst, 0755); err != nil { + return nil, fmt.Errorf("copying package script: %s", err) + } + + // Run the package script in chroot + cmd = exec.Command("chroot", mergedDir, "/bin/bash", "/tmp/packages.sh") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", "DEBIAN_FRONTEND=noninteractive"} + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("running package script: %s", err) + } + + // Clean up: remove the script, apt lists, and tmp files from upper + os.Remove(filepath.Join(upperDir, "tmp/packages.sh")) + os.RemoveAll(filepath.Join(upperDir, "var/lib/apt/lists")) + os.RemoveAll(filepath.Join(upperDir, "var/cache/apt")) + os.RemoveAll(filepath.Join(upperDir, "tmp")) + + // Remove overlayfs whiteout/opaque markers for directories we don't want to shadow + // The upper dir now contains all the changes from package installation + // We need to clean overlayfs-specific xattrs before creating squashfs + // Actually, we want the raw upper dir content — overlayfs whiteouts are + // char devices (0,0) that mksquashfs will include, but they won't work + // as intended outside overlayfs. For package installs, we shouldn't have + // deletions, so whiteouts should be minimal. Let's just remove them. + exec.Command("find", upperDir, "-type", "c", "-delete").Run() + + // Create squashfs from the upper directory (the diff) + squashfsPath := filepath.Join(workDir, "package-layer.squashfs") + cmd = exec.Command("mksquashfs", upperDir, squashfsPath, "-noappend") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("mksquashfs package layer: %s", err) + } + + // Import the squashfs layer + layer, err := e.importSquashfs(squashfsPath) + if err != nil { + return nil, err + } + + // Cache it + e.packageLayers[spec.PackageScript] = layer + fmt.Printf(" -> package layer: id=%s size=%d\n", layer.ID[:16], layer.Length) + + return layer, nil +} + // imageSpecs returns the specifications for all component images. func (e *exporter) imageSpecs() []imageSpec { return []imageSpec{ @@ -432,6 +735,12 @@ func (e *exporter) imageSpecs() []imageSpec { ExtraFiles: map[string]string{ "controller/start.sh": "/bin/start-flynn-controller", "util/ca-certs/ca-certs.pem": "/etc/ssl/certs/ca-certs.pem", + "schema/common.json": "/etc/flynn-controller/jsonschema/common.json", + "schema/error.json": "/etc/flynn-controller/jsonschema/error.json", + }, + ExtraDirs: map[string]string{ + "schema/controller": "/etc/flynn-controller/jsonschema/controller", + "schema/router": "/etc/flynn-controller/jsonschema/router", }, Entrypoint: &ct.ImageEntrypoint{ Args: []string{"/bin/start-flynn-controller"}, @@ -523,6 +832,7 @@ func (e *exporter) imageSpecs() []imageSpec { ExtraFiles: map[string]string{ "appliance/postgresql/start.sh": "/bin/start-flynn-postgres", }, + PackageScript: "appliance/postgresql/img/packages.sh", Entrypoint: &ct.ImageEntrypoint{ Args: []string{"/bin/start-flynn-postgres"}, }, @@ -539,6 +849,7 @@ func (e *exporter) imageSpecs() []imageSpec { "appliance/redis/dump.sh": "/bin/dump-flynn-redis", "appliance/redis/restore.sh": "/bin/restore-flynn-redis", }, + PackageScript: "appliance/redis/img/packages.sh", Entrypoint: &ct.ImageEntrypoint{ Args: []string{"/bin/start-flynn-redis"}, }, @@ -553,6 +864,7 @@ func (e *exporter) imageSpecs() []imageSpec { ExtraFiles: map[string]string{ "appliance/mariadb/start.sh": "/bin/start-flynn-mariadb", }, + PackageScript: "appliance/mariadb/img/packages.sh", Entrypoint: &ct.ImageEntrypoint{ Args: []string{"/bin/start-flynn-mariadb"}, }, @@ -569,6 +881,7 @@ func (e *exporter) imageSpecs() []imageSpec { "appliance/mongodb/dump.sh": "/bin/dump-flynn-mongodb", "appliance/mongodb/restore.sh": "/bin/restore-flynn-mongodb", }, + PackageScript: "appliance/mongodb/img/packages.sh", Entrypoint: &ct.ImageEntrypoint{ Args: []string{"/bin/start-flynn-mongodb"}, },