From c9f8ebbe89204fec0c3ee5239a7963fc3ae9890b Mon Sep 17 00:00:00 2001 From: borg-z Date: Thu, 20 Aug 2026 23:27:31 +0300 Subject: [PATCH 1/4] [olcedar] Add `d8 platform olcedar node add` for static immutable nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a static node running the immutable OS took five steps by hand: read the NodeGroup template, curl the machine inventory, uncomment a disk, merge the two documents with yq, and PUT the result back. Neither side could do it alone — the cluster does not know the machine's disks, the machine does not know the cluster's bootstrap token — but the CLI sees both, and unlike a file it can ask. The command reads the aggregated NodeConfigTemplate of the group and the machine's /inventory.json, asks which disk, which network and which node name, and pushes the assembled NodeConfig to the machine's maintenance port. Nothing is decided silently. The single blank disk is offered as a default but still confirmed; a disk that already holds a system is refused until the operator confirms the erase (or passes --wipe); without a terminal and without --yes the command refuses rather than choosing. A machine whose port is held by the node agent is refused by node name: a second configuration would replace the one a working node runs on. The template carries a live bootstrap token, the registry auth and the registry-packages-proxy token on every read, so the document is built in memory, never logged, and --dry-run prints it with those three fields redacted. The machine-facing HTTP contract (/whoami, /inventory.json, /config) mirrors dhctl/pkg/immutable of the deckhouse repository rather than importing it: the dhctl module would pull its whole dependency tree in for some 250 lines. Co-Authored-By: Claude Opus 5 Signed-off-by: borg-z --- internal/olcedar/cluster/cluster.go | 210 +++++++++++++ internal/olcedar/cluster/cluster_test.go | 147 +++++++++ internal/olcedar/cmd/add/add.go | 382 +++++++++++++++++++++++ internal/olcedar/cmd/add/add_test.go | 82 +++++ internal/olcedar/cmd/olcedar.go | 49 +++ internal/olcedar/machine/machine.go | 329 +++++++++++++++++++ internal/olcedar/machine/machine_test.go | 155 +++++++++ internal/olcedar/plan/plan.go | 375 ++++++++++++++++++++++ internal/olcedar/plan/plan_test.go | 254 +++++++++++++++ internal/olcedar/prompt/prompt.go | 180 +++++++++++ internal/olcedar/prompt/prompt_test.go | 117 +++++++ internal/system/cmd/system.go | 3 + 12 files changed, 2283 insertions(+) create mode 100644 internal/olcedar/cluster/cluster.go create mode 100644 internal/olcedar/cluster/cluster_test.go create mode 100644 internal/olcedar/cmd/add/add.go create mode 100644 internal/olcedar/cmd/add/add_test.go create mode 100644 internal/olcedar/cmd/olcedar.go create mode 100644 internal/olcedar/machine/machine.go create mode 100644 internal/olcedar/machine/machine_test.go create mode 100644 internal/olcedar/plan/plan.go create mode 100644 internal/olcedar/plan/plan_test.go create mode 100644 internal/olcedar/prompt/prompt.go create mode 100644 internal/olcedar/prompt/prompt_test.go diff --git a/internal/olcedar/cluster/cluster.go b/internal/olcedar/cluster/cluster.go new file mode 100644 index 000000000..a1fa3d317 --- /dev/null +++ b/internal/olcedar/cluster/cluster.go @@ -0,0 +1,210 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cluster reads the cluster's half of a node configuration: the +// NodeGroup template, the names its nodes already hold, and the node that +// appears once the machine has installed itself. +package cluster + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// GroupLabel is the label every node of a NodeGroup carries. +const GroupLabel = "node.deckhouse.io/group" + +// The NodeGroup a machine may be added to. Mirrors machineOwnedConfig of +// node-controller/src/internal/controller/nodebootstrap/template_storage.go, +// which decides the same thing on the serving side. +const ( + nodeTypeStatic = "Static" + systemTypeImmutable = "Immutable" +) + +// templateGVR is the aggregated resource that renders a NodeGroup's node +// configuration. Nothing is stored behind it: every read renders from the +// cluster as it is now, which is why the answer carries a live bootstrap token. +var templateGVR = schema.GroupVersionResource{ + Group: "templates.internal.deckhouse.io", + Version: "v1alpha1", + Resource: "nodeconfigtemplates", +} + +var nodeGroupGVR = schema.GroupVersionResource{ + Group: "deckhouse.io", + Version: "v1", + Resource: "nodegroups", +} + +// FetchTemplate reads the template of one NodeGroup. A group the cluster +// provisions itself has no template and answers 404, which says nothing about +// why, so the refusal is explained against the NodeGroup itself. +func FetchTemplate(ctx context.Context, dyn dynamic.Interface, group string) (*unstructured.Unstructured, error) { + template, err := dyn.Resource(templateGVR).Get(ctx, group, metav1.GetOptions{}) + if err == nil { + return template, nil + } + + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("read the node configuration template of %s: %w", group, err) + } + + return nil, explainMissingTemplate(ctx, dyn, group) +} + +func explainMissingTemplate(ctx context.Context, dyn dynamic.Interface, group string) error { + nodeGroup, err := dyn.Resource(nodeGroupGVR).Get(ctx, group, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return fmt.Errorf("there is no NodeGroup %q in this cluster", group) + } + + if err != nil { + return fmt.Errorf("the cluster serves no node configuration template for %s, and reading the NodeGroup to say why failed: %w", group, err) + } + + nodeType, systemType := groupTypes(nodeGroup) + + return fmt.Errorf( + "NodeGroup %s is nodeType %q, systemType %q, and only nodeType %s with systemType %s has machines that read a configuration. "+ + "Nodes of this group are provisioned by the cluster itself, so there is nothing to push to a machine", + group, nodeType, systemType, nodeTypeStatic, systemTypeImmutable) +} + +// ImmutableStaticGroups lists the NodeGroups whose machines take a +// configuration by hand. It backs the completion of --group: a group that +// cannot be added to is not offered. +func ImmutableStaticGroups(ctx context.Context, dyn dynamic.Interface) ([]string, error) { + list, err := dyn.Resource(nodeGroupGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("list NodeGroups: %w", err) + } + + var groups []string + + for i := range list.Items { + nodeType, systemType := groupTypes(&list.Items[i]) + if nodeType == nodeTypeStatic && systemType == systemTypeImmutable { + groups = append(groups, list.Items[i].GetName()) + } + } + + return groups, nil +} + +func groupTypes(nodeGroup *unstructured.Unstructured) (string, string) { + nodeType, _, _ := unstructured.NestedString(nodeGroup.Object, "spec", "nodeType") + systemType, _, _ := unstructured.NestedString(nodeGroup.Object, "spec", "systemType") + + return nodeType, systemType +} + +// FreeNodeName is the first - no node of the group holds. A machine +// that was handed a configuration but has not registered yet holds no name +// here, so the name is checked again right before the push. +func FreeNodeName(ctx context.Context, kube kubernetes.Interface, group string) (string, error) { + nodes, err := kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: GroupLabel + "=" + group}) + if err != nil { + return "", fmt.Errorf("list the nodes of %s: %w", group, err) + } + + taken := make(map[int]bool, len(nodes.Items)) + + for i := range nodes.Items { + if index, ok := nodeIndex(nodes.Items[i].Name, group); ok { + taken[index] = true + } + } + + for index := 0; ; index++ { + if !taken[index] { + return fmt.Sprintf("%s-%d", group, index), nil + } + } +} + +func nodeIndex(nodeName, group string) (int, bool) { + suffix, found := strings.CutPrefix(nodeName, group+"-") + if !found { + return 0, false + } + + index, err := strconv.Atoi(suffix) + if err != nil || index < 0 { + return 0, false + } + + return index, true +} + +// NodeNameByAddress names the node that registered with this address, so a +// machine that already is one can be refused by name rather than by address. +func NodeNameByAddress(ctx context.Context, kube kubernetes.Interface, host string) string { + nodes, err := kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return "" + } + + for i := range nodes.Items { + for _, address := range nodes.Items[i].Status.Addresses { + if address.Type == corev1.NodeInternalIP && address.Address == host { + return nodes.Items[i].Name + } + } + } + + return "" +} + +// NodeExists answers whether the cluster already holds a node under this name. +func NodeExists(ctx context.Context, kube kubernetes.Interface, name string) (bool, error) { + _, err := kube.CoreV1().Nodes().Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("read node %s: %w", name, err) + } + + return true, nil +} + +// WaitForNode waits until the node registers. Registration is what this command +// can promise: it means the machine installed itself and its kubelet reached +// the cluster. Readiness comes later, from the modules rolled onto the node. +func WaitForNode(ctx context.Context, kube kubernetes.Interface, name string, timeout time.Duration) error { + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + return NodeExists(ctx, kube, name) + }) + if err != nil { + return fmt.Errorf("wait for node %s to register: %w", name, err) + } + + return nil +} diff --git a/internal/olcedar/cluster/cluster_test.go b/internal/olcedar/cluster/cluster_test.go new file mode 100644 index 000000000..89772f632 --- /dev/null +++ b/internal/olcedar/cluster/cluster_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" +) + +func node(name, group string, addresses ...string) *corev1.Node { + status := corev1.NodeStatus{} + for _, address := range addresses { + status.Addresses = append(status.Addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address}) + } + + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{GroupLabel: group}}, + Status: status, + } +} + +func nodeGroup(name, nodeType, systemType string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "deckhouse.io/v1", + "kind": "NodeGroup", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"nodeType": nodeType, "systemType": systemType}, + }} +} + +func template(name string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "templates.internal.deckhouse.io/v1alpha1", + "kind": "NodeConfigTemplate", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"nodeName": ""}, + }} +} + +func dynamicClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { + scheme := runtime.NewScheme() + + return dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + templateGVR: "NodeConfigTemplateList", + nodeGroupGVR: "NodeGroupList", + }, objects...) +} + +// The template of an existing immutable static group is what the command builds +// the document out of. +func TestFetchTemplateReadsTheGroupTemplate(t *testing.T) { + got, err := FetchTemplate(context.Background(), dynamicClient(template("worker")), "worker") + require.NoError(t, err) + require.Equal(t, "worker", got.GetName()) +} + +// A 404 says nothing about why, so the refusal names the reason the operator +// can act on rather than "not found". +func TestFetchTemplateExplainsAGroupOfTheWrongType(t *testing.T) { + _, err := FetchTemplate(context.Background(), dynamicClient(nodeGroup("worker", "CloudEphemeral", "")), "worker") + require.ErrorContains(t, err, "CloudEphemeral") + require.ErrorContains(t, err, "provisioned by the cluster itself") +} + +func TestFetchTemplateExplainsAMissingGroup(t *testing.T) { + _, err := FetchTemplate(context.Background(), dynamicClient(), "worker") + require.ErrorContains(t, err, `there is no NodeGroup "worker"`) +} + +func TestImmutableStaticGroupsOffersOnlyGroupsThatTakeAMachine(t *testing.T) { + groups, err := ImmutableStaticGroups(context.Background(), dynamicClient( + nodeGroup("worker", "Static", "Immutable"), + nodeGroup("legacy", "Static", ""), + nodeGroup("cloud", "CloudEphemeral", ""), + )) + require.NoError(t, err) + require.Equal(t, []string{"worker"}, groups) +} + +// The default name has to be free: a group holding worker-0 gets worker-1. +func TestFreeNodeNameTakesTheFirstFreeNumber(t *testing.T) { + kube := fake.NewSimpleClientset(node("worker-0", "worker")) + + name, err := FreeNodeName(context.Background(), kube, "worker") + require.NoError(t, err) + require.Equal(t, "worker-1", name) +} + +func TestFreeNodeNameFillsAHoleAndIgnoresOtherGroups(t *testing.T) { + kube := fake.NewSimpleClientset( + node("worker-0", "worker"), + node("worker-2", "worker"), + node("worker-gpu-1", "worker-gpu"), + ) + + name, err := FreeNodeName(context.Background(), kube, "worker") + require.NoError(t, err) + require.Equal(t, "worker-1", name) +} + +func TestFreeNodeNameStartsAtZeroForAnEmptyGroup(t *testing.T) { + name, err := FreeNodeName(context.Background(), fake.NewSimpleClientset(), "worker") + require.NoError(t, err) + require.Equal(t, "worker-0", name) +} + +func TestNodeNameByAddressNamesTheNodeHoldingTheAddress(t *testing.T) { + kube := fake.NewSimpleClientset(node("worker-0", "worker", "10.12.4.55")) + + require.Equal(t, "worker-0", NodeNameByAddress(context.Background(), kube, "10.12.4.55")) + require.Empty(t, NodeNameByAddress(context.Background(), kube, "10.12.4.56")) +} + +func TestNodeExists(t *testing.T) { + kube := fake.NewSimpleClientset(node("worker-0", "worker")) + + exists, err := NodeExists(context.Background(), kube, "worker-0") + require.NoError(t, err) + require.True(t, exists) + + exists, err = NodeExists(context.Background(), kube, "worker-1") + require.NoError(t, err) + require.False(t, exists) +} diff --git a/internal/olcedar/cmd/add/add.go b/internal/olcedar/cmd/add/add.go new file mode 100644 index 000000000..8f97f5279 --- /dev/null +++ b/internal/olcedar/cmd/add/add.go @@ -0,0 +1,382 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package add introduces a machine waiting on its maintenance port into the +// cluster as a static node with an immutable OS. +package add + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "golang.org/x/term" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/deckhouse/deckhouse-cli/internal/olcedar/cluster" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/machine" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/plan" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/prompt" + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" +) + +const ( + networkDHCP = "dhcp" + networkStatic = "static" +) + +var addLong = templates.LongDesc(` +Add a machine to the cluster as a static node running an immutable OS (olcedar). + +The machine waits for its configuration on port 50000, and the cluster renders +the rest of that configuration for the NodeGroup. This command reads both +halves, asks about what only an operator can decide — the disk, the network and +the node name — and hands the result to the machine. + +Only a NodeGroup with nodeType Static and systemType Immutable has machines that +read a configuration. Nodes of any other group are provisioned by the cluster +itself, and --group offers no such group. + +© Flant JSC 2026`) + +type options struct { + group string + name string + diskSelector string + network string + networkInterface string + wipe bool + yes bool + dryRun bool + wait bool + waitTimeout time.Duration +} + +func NewCommand() *cobra.Command { + opts := &options{} + + addCmd := &cobra.Command{ + Use: "add
", + Short: "Add a static node running an immutable OS", + Long: addLong, + Args: cobra.ExactArgs(1), + Example: ` # Ask about the disk and the network, then add the machine as worker-1. + d8 platform olcedar node add 10.12.4.55 --group worker + + # Decide everything up front, for a script. + d8 platform olcedar node add 10.12.4.55 --group worker \ + --name worker-1 --disk-selector serial=S3Z8NB0K700002 --network dhcp --yes`, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd, args[0], opts) + }, + } + + flags := addCmd.Flags() + flags.StringVar(&opts.group, "group", "", "NodeGroup to add the machine to") + flags.StringVar(&opts.name, "name", "", "Name to register the node under (default -)") + flags.StringVar(&opts.diskSelector, "disk-selector", "", "Disk to install onto, as key=value (serial, wwid, name, busPath, model)") + flags.StringVar(&opts.network, "network", "", "Network configuration: dhcp or static (default asks, dhcp)") + flags.StringVar(&opts.networkInterface, "network-interface", "", "Interface to configure, when the address reaches the machine through a forward") + flags.BoolVar(&opts.wipe, "wipe", false, "Allow the install to erase a disk that is not blank") + flags.BoolVar(&opts.yes, "yes", false, "Answer every question with its default, and refuse where there is no default") + flags.BoolVar(&opts.dryRun, "dry-run", false, "Print the document with its secrets redacted instead of pushing it") + flags.BoolVar(&opts.wait, "wait", true, "Wait for the node to register in the cluster") + flags.DurationVar(&opts.waitTimeout, "wait-timeout", 20*time.Minute, "How long to wait for the node to register") + + if err := addCmd.MarkFlagRequired("group"); err != nil { + panic(err) + } + + if err := addCmd.RegisterFlagCompletionFunc("group", completeGroups); err != nil { + panic(err) + } + + return addCmd +} + +func completeGroups(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + dyn, err := utilk8s.NewDynamicClient(cmd) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) + defer cancel() + + groups, err := cluster.ImmutableStaticGroups(ctx, dyn) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + return groups, cobra.ShellCompDirectiveNoFileComp +} + +func run(cmd *cobra.Command, target string, opts *options) error { + if err := validate(opts, term.IsTerminal(int(os.Stdin.Fd()))); err != nil { + return err + } + + ctx := cmd.Context() + address := machine.Address(target) + p := prompt.New(cmd.InOrStdin(), cmd.OutOrStdout(), opts.yes) + + kube, dyn, err := clients(cmd) + if err != nil { + return err + } + + if err := checkMachineIsNotANode(ctx, kube, address); err != nil { + return err + } + + template, err := cluster.FetchTemplate(ctx, dyn, opts.group) + if err != nil { + return err + } + + inventory, err := machine.FetchInventory(ctx, address) + if err != nil { + return err + } + + choices, err := decide(ctx, p, kube, inventory, machine.Host(address), opts) + if err != nil { + return err + } + + document, err := plan.BuildDocument(template, *choices) + if err != nil { + return err + } + + if opts.dryRun { + return printRedacted(cmd, document) + } + + if err := refuseTakenName(ctx, kube, choices.NodeName); err != nil { + return err + } + + if err := machine.PushNodeConfig(ctx, address, document); err != nil { + return err + } + + p.Printf("\n%s accepted the configuration and is installing itself as %s.\n", address, choices.NodeName) + + if !opts.wait { + return nil + } + + return waitForNode(ctx, p, kube, choices.NodeName, opts.waitTimeout) +} + +func validate(opts *options, interactive bool) error { + switch opts.network { + case "", networkDHCP, networkStatic: + default: + return fmt.Errorf("--network is %q, and it takes %s or %s", opts.network, networkDHCP, networkStatic) + } + + if opts.yes || interactive { + return nil + } + + return errors.New("this command asks which disk to install onto and how the node reaches the network, " + + "and there is no terminal to ask on: run it with --yes, and name what it would have asked with " + + "--name, --disk-selector, --wipe, --network and --network-interface") +} + +func clients(cmd *cobra.Command) (kubernetes.Interface, dynamic.Interface, error) { + kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + if err != nil { + return nil, nil, fmt.Errorf("read the kubeconfig flag: %w", err) + } + + contextName, err := cmd.Flags().GetString("context") + if err != nil { + return nil, nil, fmt.Errorf("read the context flag: %w", err) + } + + restConfig, kube, err := utilk8s.SetupK8sClientSet(kubeconfigPath, contextName) + if err != nil { + return nil, nil, fmt.Errorf("set up the Kubernetes client: %w", err) + } + + dyn, err := dynamic.NewForConfig(restConfig) + if err != nil { + return nil, nil, fmt.Errorf("set up the dynamic Kubernetes client: %w", err) + } + + return kube, dyn, nil +} + +// checkMachineIsNotANode refuses a machine whose port is held by the agent of +// an installed node: a second configuration would take a working node down. +func checkMachineIsNotANode(ctx context.Context, kube kubernetes.Interface, address string) error { + who, err := machine.Whoami(ctx, address) + if err != nil { + return err + } + + switch who { + case machine.WhoamiInstaller: + return nil + case machine.WhoamiAgent: + return fmt.Errorf("%s is already %s: its port is held by the node agent, and a second configuration "+ + "would replace the configuration that node runs on", address, nodeDescription(ctx, kube, address)) + default: + return fmt.Errorf("%s answered %q to /whoami, which is neither %s nor %s: whatever holds port %s is not olcedar", + address, who, machine.WhoamiInstaller, machine.WhoamiAgent, machine.MaintenancePort) + } +} + +func nodeDescription(ctx context.Context, kube kubernetes.Interface, address string) string { + if name := cluster.NodeNameByAddress(ctx, kube, machine.Host(address)); name != "" { + return "node " + name + } + + return "a node of this cluster" +} + +func decide( + ctx context.Context, + p *prompt.Prompt, + kube kubernetes.Interface, + inventory *machine.Inventory, + host string, + opts *options, +) (*plan.Choices, error) { + selector, err := parseSelector(opts.diskSelector) + if err != nil { + return nil, err + } + + disk, wipe, err := plan.ChooseDisk(p, inventory, selector, opts.wipe) + if err != nil { + return nil, err + } + + iface, err := plan.ChooseInterface(p, inventory, host, opts.networkInterface) + if err != nil { + return nil, err + } + + static, err := chooseStatic(p, iface, opts.network) + if err != nil { + return nil, err + } + + name, err := chooseName(ctx, p, kube, opts) + if err != nil { + return nil, err + } + + return &plan.Choices{ + NodeName: name, + Disk: disk, + Selector: machine.SelectorFor(disk), + Wipe: wipe, + Interface: iface, + StaticAddress: static, + }, nil +} + +func parseSelector(raw string) (machine.Selector, error) { + if raw == "" { + return nil, nil + } + + return machine.ParseSelector(raw) +} + +func chooseStatic(p *prompt.Prompt, iface machine.Interface, network string) (bool, error) { + if network == networkDHCP { + return false, nil + } + + if network == networkStatic { + if len(iface.Addresses) == 0 { + return false, fmt.Errorf("--network %s pins the addresses interface %s holds, and it holds none", + networkStatic, iface.Name) + } + + return true, nil + } + + if len(iface.Addresses) == 0 { + return false, nil + } + + return p.Confirm(fmt.Sprintf("Interface %s holds %s. Pin it statically instead of using DHCP?", + iface.Name, iface.Addresses[0]), false) +} + +func chooseName(ctx context.Context, p *prompt.Prompt, kube kubernetes.Interface, opts *options) (string, error) { + if opts.name != "" { + return opts.name, nil + } + + free, err := cluster.FreeNodeName(ctx, kube, opts.group) + if err != nil { + return "", err + } + + return p.Line("Node name", free) +} + +func refuseTakenName(ctx context.Context, kube kubernetes.Interface, name string) error { + taken, err := cluster.NodeExists(ctx, kube, name) + if err != nil { + return err + } + + if taken { + return fmt.Errorf("node %s is already in this cluster: pick another name with --name", name) + } + + return nil +} + +func printRedacted(cmd *cobra.Command, document []byte) error { + redacted, err := plan.Redact(document) + if err != nil { + return err + } + + fmt.Fprintf(cmd.OutOrStdout(), "%s", redacted) + + return nil +} + +func waitForNode(ctx context.Context, p *prompt.Prompt, kube kubernetes.Interface, name string, timeout time.Duration) error { + started := time.Now() + + p.Printf("Waiting for %s to register, up to %s.\n", name, timeout) + + if err := cluster.WaitForNode(ctx, kube, name, timeout); err != nil { + return err + } + + p.Printf("%s registered in %s. It becomes Ready once the cluster rolls its modules onto it:\n d8 k wait --for=condition=Ready node/%s\n", + name, time.Since(started).Round(time.Second), name) + + return nil +} diff --git a/internal/olcedar/cmd/add/add_test.go b/internal/olcedar/cmd/add/add_test.go new file mode 100644 index 000000000..b0afebcfb --- /dev/null +++ b/internal/olcedar/cmd/add/add_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package add + +import ( + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/internal/olcedar/machine" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/prompt" +) + +func assuming() *prompt.Prompt { + return prompt.New(strings.NewReader(""), io.Discard, true) +} + +// Nothing is chosen for the operator: with no terminal to ask on and no --yes, +// the command refuses instead of picking a disk itself. +func TestValidateRefusesWithoutATerminalAndWithoutYes(t *testing.T) { + err := validate(&options{}, false) + require.ErrorContains(t, err, "--yes") + require.ErrorContains(t, err, "--disk-selector") + + require.NoError(t, validate(&options{yes: true}, false)) + require.NoError(t, validate(&options{}, true)) +} + +func TestValidateRefusesAnUnknownNetworkMode(t *testing.T) { + require.ErrorContains(t, validate(&options{network: "bridge", yes: true}, false), "--network") + require.NoError(t, validate(&options{network: networkDHCP, yes: true}, false)) + require.NoError(t, validate(&options{network: networkStatic, yes: true}, false)) +} + +func TestParseSelectorPassesAnEmptyFlagThrough(t *testing.T) { + selector, err := parseSelector("") + require.NoError(t, err) + require.Nil(t, selector) + + selector, err = parseSelector("serial=S3Z8NB0K700002") + require.NoError(t, err) + require.Equal(t, machine.Selector{"serial": "S3Z8NB0K700002"}, selector) +} + +func TestChooseStaticFollowsTheFlag(t *testing.T) { + iface := machine.Interface{Name: "eno1", Addresses: []string{"10.12.4.55/24"}} + + static, err := chooseStatic(assuming(), iface, networkDHCP) + require.NoError(t, err) + require.False(t, static) + + static, err = chooseStatic(assuming(), iface, networkStatic) + require.NoError(t, err) + require.True(t, static) + + _, err = chooseStatic(assuming(), machine.Interface{Name: "eno2"}, networkStatic) + require.ErrorContains(t, err, "holds none") +} + +// Left unsaid, the network stays on DHCP: pinning an address is offered, never +// assumed. +func TestChooseStaticDefaultsToDHCP(t *testing.T) { + static, err := chooseStatic(assuming(), machine.Interface{Name: "eno1", Addresses: []string{"10.12.4.55/24"}}, "") + require.NoError(t, err) + require.False(t, static) +} diff --git a/internal/olcedar/cmd/olcedar.go b/internal/olcedar/cmd/olcedar.go new file mode 100644 index 000000000..8b9f22c69 --- /dev/null +++ b/internal/olcedar/cmd/olcedar.go @@ -0,0 +1,49 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cmd holds the olcedar command group: what an operator does to a +// machine running the immutable OS from the cluster side. +package cmd + +import ( + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/deckhouse/deckhouse-cli/internal/olcedar/cmd/add" +) + +var olcedarLong = templates.LongDesc(` +Operate machines running olcedar, the immutable OS of the platform. + +© Flant JSC 2026`) + +func NewCommand() *cobra.Command { + olcedarCmd := &cobra.Command{ + Use: "olcedar", + Short: "Operate machines running the immutable OS", + Long: olcedarLong, + } + + nodeCmd := &cobra.Command{ + Use: "node", + Short: "Operate the nodes such machines become", + } + + nodeCmd.AddCommand(add.NewCommand()) + olcedarCmd.AddCommand(nodeCmd) + + return olcedarCmd +} diff --git a/internal/olcedar/machine/machine.go b/internal/olcedar/machine/machine.go new file mode 100644 index 000000000..fce976bb1 --- /dev/null +++ b/internal/olcedar/machine/machine.go @@ -0,0 +1,329 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package machine talks to a machine waiting for its node configuration on the +// maintenance port. The wire contract mirrors dhctl/pkg/immutable of the +// deckhouse repository (inventory.go, push.go, constants.go), which the +// installer uses for the same three calls. +package machine + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "path" + "slices" + "strings" + "time" +) + +// MaintenancePort is where olcedar-init waits for a node configuration. It +// closes the moment a document is accepted. +const MaintenancePort = "50000" + +const ( + inventoryPath = "/inventory.json" + configPath = "/config" + whoamiPath = "/whoami" +) + +// The two answers /whoami gives. +const ( + WhoamiInstaller = "installer" + WhoamiAgent = "agent" +) + +const ( + whoamiTimeout = 5 * time.Second + // pushTimeout bounds one PUT: the machine writes the document to its config + // partition before answering, which is a disk write. + pushTimeout = 30 * time.Second +) + +// maxErrorBody caps how much of a failing response is quoted back. +const maxErrorBody = 512 + +// Disk states the machine reports, which the size cannot say. +const ( + StateBlank = "blank" + StateFormatted = "formatted" + StateSystemLayout = "system-layout" +) + +// ErrNoInventory means the machine serves no inventory: an image too old to +// have the endpoint answers 404 to it. +var ErrNoInventory = errors.New("the machine serves no inventory") + +// ErrAlreadyConfigured means the port is held by the agent of an installed +// node, which will not take a second configuration without a maintenance token. +var ErrAlreadyConfigured = errors.New("the machine is already a node") + +// Inventory is what a machine says about itself before anything is installed. +// The shape is the wire contract: Inventory in images/init/src/0.1/inventory.go +// of the initramfs repository. +type Inventory struct { + Disks []Disk `json:"disks"` + Interfaces []Interface `json:"interfaces"` +} + +type Disk struct { + Name string `json:"name"` + Size uint64 `json:"size"` + Model string `json:"model"` + Vendor string `json:"vendor"` + Serial string `json:"serial"` + WWID string `json:"wwid"` + Rotational bool `json:"rotational"` + Transport string `json:"transport"` + ByPath string `json:"byPath"` + ByID []string `json:"byId"` + BusPath string `json:"busPath"` + State string `json:"state"` + Partitions []Partition `json:"partitions"` +} + +type Partition struct { + Name string `json:"name"` + Size uint64 `json:"size"` + FSType string `json:"fsType"` + Label string `json:"label"` +} + +type Interface struct { + Name string `json:"name"` + MAC string `json:"mac"` + Link string `json:"link"` + Addresses []string `json:"addresses"` + Gateway string `json:"gateway"` + Source string `json:"source"` +} + +// Address adds the maintenance port to a bare host, and keeps one already +// written with a port. +func Address(hostOrAddress string) string { + if _, _, err := net.SplitHostPort(hostOrAddress); err == nil { + return hostOrAddress + } + + return net.JoinHostPort(hostOrAddress, MaintenancePort) +} + +// Host is the address without its port, as the cluster spells a node address. +func Host(address string) string { + host, _, err := net.SplitHostPort(address) + if err != nil { + return address + } + + return host +} + +// Whoami tells which of the two servers holds the maintenance port: the +// installer waiting for a configuration, or the agent of an installed node. +func Whoami(ctx context.Context, address string) (string, error) { + body, err := get(ctx, address, whoamiPath, whoamiTimeout) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(body)), nil +} + +// FetchInventory reads what the machine says about its own hardware. An image +// too old to serve the endpoint answers 404, which is ErrNoInventory: there is +// then nothing to pick a disk out of. +func FetchInventory(ctx context.Context, address string) (*Inventory, error) { + body, err := get(ctx, address, inventoryPath, pushTimeout) + if err != nil { + return nil, err + } + + inventory := &Inventory{} + if err := json.Unmarshal(body, inventory); err != nil { + return nil, fmt.Errorf("read the inventory of %s: %w", address, err) + } + + return inventory, nil +} + +// PushNodeConfig hands the machine the document it boots from. The endpoint is +// unauthenticated by design — the machine holds no secret at this point — so +// the caller answers for the network the address lives on. +func PushNodeConfig(ctx context.Context, address string, document []byte) error { + request, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+address+configPath, bytes.NewReader(document)) + if err != nil { + return fmt.Errorf("build the push request for %s: %w", address, err) + } + + request.Header.Set("Content-Type", "application/yaml") + + response, err := do(request, pushTimeout) + if err != nil { + return fmt.Errorf("push the node configuration to %s: %w", address, err) + } + defer func() { _ = response.Body.Close() }() + + if response.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("push the node configuration to %s: %w", address, ErrAlreadyConfigured) + } + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("push the node configuration to %s: %s: %s", address, response.Status, errorBody(response)) + } + + return nil +} + +func get(ctx context.Context, address, urlPath string, timeout time.Duration) ([]byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address+urlPath, nil) + if err != nil { + return nil, fmt.Errorf("build the request for %s%s: %w", address, urlPath, err) + } + + response, err := do(request, timeout) + if err != nil { + return nil, fmt.Errorf("read %s%s: %w", address, urlPath, err) + } + defer func() { _ = response.Body.Close() }() + + if response.StatusCode == http.StatusNotFound && urlPath == inventoryPath { + return nil, fmt.Errorf("%w: %s answered 404 to %s, which an image built before the endpoint does", ErrNoInventory, address, urlPath) + } + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s to %s: %s", address, response.Status, urlPath, errorBody(response)) + } + + body, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("read the answer of %s%s: %w", address, urlPath, err) + } + + return body, nil +} + +func do(request *http.Request, timeout time.Duration) (*http.Response, error) { + client := &http.Client{Timeout: timeout} + defer client.CloseIdleConnections() + + return client.Do(request) +} + +func errorBody(response *http.Response) string { + body, err := io.ReadAll(io.LimitReader(response.Body, maxErrorBody)) + if err != nil { + return "the refusal could not be read: " + err.Error() + } + + return string(bytes.TrimSpace(body)) +} + +// Selector picks a disk by the attributes the machine reports. Every field set +// must match, shell-style patterns included, the way DiskSelector of the +// NodeConfig CRD is matched on the node itself. +type Selector map[string]string + +// SelectorKeys are the attributes a selector may name, in the spelling the +// NodeConfig spec.storage.diskSelector uses. +var SelectorKeys = []string{"serial", "wwid", "name", "busPath", "model"} + +// ParseSelector reads a "key=value" pair into a selector. +func ParseSelector(raw string) (Selector, error) { + key, value, found := strings.Cut(raw, "=") + if !found { + return nil, fmt.Errorf("disk selector %q is not key=value, where key is one of %s", raw, strings.Join(SelectorKeys, ", ")) + } + + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !slices.Contains(SelectorKeys, key) { + return nil, fmt.Errorf("disk selector key %q is none of %s", key, strings.Join(SelectorKeys, ", ")) + } + + if value == "" { + return nil, fmt.Errorf("disk selector %q has an empty value, which matches nothing", raw) + } + + return Selector{key: value}, nil +} + +// Match lists the disks the selector describes. Mirrors matchDisk in +// images/init/src/0.1/disk.go of the initramfs repository: the machine matches +// these fields itself and the two must not disagree. +func (s Selector) Match(disks []Disk) ([]Disk, error) { + var matched []Disk + + for _, disk := range disks { + ok, err := s.matches(disk) + if err != nil { + return nil, err + } + + if ok { + matched = append(matched, disk) + } + } + + return matched, nil +} + +func (s Selector) matches(disk Disk) (bool, error) { + for key, pattern := range s { + ok, err := path.Match(pattern, attribute(key, disk)) + if err != nil { + return false, fmt.Errorf("disk selector %s=%q is not a valid pattern: %w", key, pattern, err) + } + + if !ok { + return false, nil + } + } + + return true, nil +} + +func attribute(key string, disk Disk) string { + switch key { + case "serial": + return disk.Serial + case "wwid": + return disk.WWID + case "name": + return disk.Name + case "busPath": + return disk.BusPath + case "model": + return disk.Model + default: + panic("unknown disk selector key " + key) + } +} + +// SelectorFor names the disk by the most stable attribute it reports, so the +// document survives the machine renaming sda to sdb between boots. +func SelectorFor(disk Disk) Selector { + for _, key := range []string{"wwid", "serial", "busPath", "name"} { + if value := attribute(key, disk); value != "" { + return Selector{key: value} + } + } + + return Selector{"name": disk.Name} +} diff --git a/internal/olcedar/machine/machine_test.go b/internal/olcedar/machine/machine_test.go new file mode 100644 index 000000000..4a5383557 --- /dev/null +++ b/internal/olcedar/machine/machine_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machine + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const inventoryBody = `{ + "disks": [ + {"name": "nvme0n1", "size": 512110190592, "model": "MZVL2512", "serial": "S3Z8NB0K700002", "wwid": "eui.0025", "state": "blank"}, + {"name": "sda", "size": 2000398934016, "model": "ST2000DM008", "serial": "ZA20ABCD", "state": "system-layout"} + ], + "interfaces": [ + {"name": "eno1", "mac": "aa:bb:cc:dd:ee:01", "link": "up", "addresses": ["10.12.4.55/24"], "gateway": "10.12.4.1"} + ] +}` + +func serve(t *testing.T, handler http.HandlerFunc) string { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + return strings.TrimPrefix(server.URL, "http://") +} + +func TestWhoamiReadsTheAnswer(t *testing.T) { + address := serve(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, whoamiPath, r.URL.Path) + _, _ = io.WriteString(w, WhoamiAgent+"\n") + }) + + who, err := Whoami(context.Background(), address) + require.NoError(t, err) + require.Equal(t, WhoamiAgent, who) +} + +func TestFetchInventoryParsesTheWireContract(t *testing.T) { + address := serve(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, inventoryPath, r.URL.Path) + _, _ = io.WriteString(w, inventoryBody) + }) + + inventory, err := FetchInventory(context.Background(), address) + require.NoError(t, err) + require.Len(t, inventory.Disks, 2) + require.Equal(t, StateBlank, inventory.Disks[0].State) + require.Equal(t, "S3Z8NB0K700002", inventory.Disks[0].Serial) + require.Equal(t, StateSystemLayout, inventory.Disks[1].State) + require.Equal(t, []string{"10.12.4.55/24"}, inventory.Interfaces[0].Addresses) +} + +// An image built before the endpoint answers 404, and this command has nothing +// to pick a disk out of then. +func TestFetchInventoryReportsAnImageWithoutTheEndpoint(t *testing.T) { + address := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + _, err := FetchInventory(context.Background(), address) + require.ErrorIs(t, err, ErrNoInventory) +} + +func TestPushNodeConfigSendsTheDocument(t *testing.T) { + var got []byte + + address := serve(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodPut, r.Method) + require.Equal(t, configPath, r.URL.Path) + require.Equal(t, "application/yaml", r.Header.Get("Content-Type")) + + got, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + }) + + require.NoError(t, PushNodeConfig(context.Background(), address, []byte("kind: NodeConfig\n"))) + require.Equal(t, "kind: NodeConfig\n", string(got)) +} + +func TestPushNodeConfigReportsAnInstalledNode(t *testing.T) { + address := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + + err := PushNodeConfig(context.Background(), address, []byte("kind: NodeConfig\n")) + require.ErrorIs(t, err, ErrAlreadyConfigured) +} + +func TestAddressKeepsAPortAndAddsTheDefaultOne(t *testing.T) { + require.Equal(t, "10.12.4.55:"+MaintenancePort, Address("10.12.4.55")) + require.Equal(t, "10.12.4.55:9000", Address("10.12.4.55:9000")) + require.Equal(t, "10.12.4.55", Host("10.12.4.55:50000")) +} + +func TestParseSelectorRefusesWhatMatchesNothing(t *testing.T) { + selector, err := ParseSelector("serial=S3Z8NB0K700002") + require.NoError(t, err) + require.Equal(t, Selector{"serial": "S3Z8NB0K700002"}, selector) + + _, err = ParseSelector("serial") + require.Error(t, err) + + _, err = ParseSelector("colour=blue") + require.Error(t, err) + + _, err = ParseSelector("serial=") + require.Error(t, err) +} + +func TestSelectorMatchesByGlob(t *testing.T) { + disks := []Disk{ + {Name: "nvme0n1", Serial: "S3Z8NB0K700002", WWID: "eui.0025"}, + {Name: "sda", Serial: "ZA20ABCD"}, + } + + matched, err := Selector{"serial": "S3Z*"}.Match(disks) + require.NoError(t, err) + require.Len(t, matched, 1) + require.Equal(t, "nvme0n1", matched[0].Name) + + matched, err = Selector{"name": "*"}.Match(disks) + require.NoError(t, err) + require.Len(t, matched, 2) +} + +// The document has to survive the machine renaming sda to sdb between boots, +// so the most stable attribute the disk reports is the one written down. +func TestSelectorForPrefersTheStablestAttribute(t *testing.T) { + require.Equal(t, Selector{"wwid": "eui.0025"}, SelectorFor(Disk{Name: "nvme0n1", Serial: "S3Z", WWID: "eui.0025"})) + require.Equal(t, Selector{"serial": "S3Z"}, SelectorFor(Disk{Name: "nvme0n1", Serial: "S3Z"})) + require.Equal(t, Selector{"busPath": "pci-0000:00"}, SelectorFor(Disk{Name: "sda", BusPath: "pci-0000:00"})) + require.Equal(t, Selector{"name": "sda"}, SelectorFor(Disk{Name: "sda"})) +} diff --git a/internal/olcedar/plan/plan.go b/internal/olcedar/plan/plan.go new file mode 100644 index 000000000..e3cfc9e16 --- /dev/null +++ b/internal/olcedar/plan/plan.go @@ -0,0 +1,375 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package plan turns what the machine reports and what the operator answers +// into the one document the machine boots from. +package plan + +import ( + "errors" + "fmt" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + "github.com/deckhouse/deckhouse-cli/internal/olcedar/machine" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/prompt" +) + +// The document a machine reads. Mirrors payloadAPIVersion and nodeConfigKind of +// dhctl/pkg/immutable/constants.go, which spells the same pair when the +// installer builds one. +const ( + documentAPIVersion = "internal.deckhouse.io/v1alpha1" + documentKind = "NodeConfig" +) + +// redactedPaths are the fields a template carries live on every read. They are +// blanked before a document is ever shown to anyone. +var redactedPaths = [][]string{ + {"spec", "kubelet", "bootstrapToken"}, + {"spec", "registry", "auth"}, + {"spec", "registryPackagesProxyAccessTokenB64"}, +} + +// Redacted is what a secret reads as once it is not printed. +const Redacted = "REDACTED" + +// Choices is everything the operator decided about this machine. +type Choices struct { + NodeName string + Disk machine.Disk + Selector machine.Selector + Wipe bool + Interface machine.Interface + // StaticAddress pins the addresses the interface currently holds instead of + // leaving it on DHCP. + StaticAddress bool +} + +// ChooseDisk settles which disk the OS is installed onto, and whether the +// install may erase what is on it. A selector given on the command line has to +// resolve to exactly one disk; without one the operator picks from the list. +func ChooseDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine.Selector, wipe bool) (machine.Disk, bool, error) { + if len(inventory.Disks) == 0 { + return machine.Disk{}, false, errors.New("this machine reports no disks, so there is nothing to install onto") + } + + disk, err := pickDisk(p, inventory, selector) + if err != nil { + return machine.Disk{}, false, err + } + + if disk.State == machine.StateBlank { + return disk, false, nil + } + + if wipe { + return disk, true, nil + } + + confirmed, err := p.Confirm(fmt.Sprintf( + "Disk %s is %s and the install will erase it. Continue?", disk.Name, disk.State), false) + if err != nil { + return machine.Disk{}, false, err + } + + if !confirmed { + return machine.Disk{}, false, fmt.Errorf( + "disk %s is %s, not %s: erasing it has to be confirmed, or allowed with --wipe", + disk.Name, disk.State, machine.StateBlank) + } + + return disk, true, nil +} + +func pickDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine.Selector) (machine.Disk, error) { + if len(selector) > 0 { + return matchOne(selector, inventory.Disks) + } + + options := make([]string, 0, len(inventory.Disks)) + for _, disk := range inventory.Disks { + options = append(options, describeDisk(disk)) + } + + index, err := p.Choose("Disks this machine reports:", options, defaultDisk(inventory.Disks)) + if err != nil { + return machine.Disk{}, fmt.Errorf("%w. Name the disk with --disk-selector, e.g. --disk-selector serial=%s", + err, firstNonEmptySerial(inventory.Disks)) + } + + return inventory.Disks[index], nil +} + +func matchOne(selector machine.Selector, disks []machine.Disk) (machine.Disk, error) { + matched, err := selector.Match(disks) + if err != nil { + return machine.Disk{}, err + } + + switch len(matched) { + case 1: + return matched[0], nil + case 0: + return machine.Disk{}, fmt.Errorf("the disk selector matches no disk of this machine, which has:\n%s", describeDisks(disks)) + default: + return machine.Disk{}, fmt.Errorf("the disk selector matches %d disks and only one can hold the system:\n%s", + len(matched), describeDisks(matched)) + } +} + +// defaultDisk offers the one blank disk, and nothing when there is a choice to +// make: a disk holding data is never picked for the operator. +func defaultDisk(disks []machine.Disk) int { + found := prompt.NoDefault + + for i, disk := range disks { + if disk.State != machine.StateBlank { + continue + } + + if found != prompt.NoDefault { + return prompt.NoDefault + } + + found = i + } + + return found +} + +// ChooseInterface settles which NIC the node configures. The interface the CLI +// reached the machine on is proven by the connection itself, so it is the +// default; an address that belongs to none of them (a port forward, a NAT) has +// to be answered for. +func ChooseInterface(p *prompt.Prompt, inventory *machine.Inventory, host, named string) (machine.Interface, error) { + if len(inventory.Interfaces) == 0 { + return machine.Interface{}, errors.New("this machine reports no interfaces, so there is nothing to configure") + } + + if named != "" { + index := slices.IndexFunc(inventory.Interfaces, func(i machine.Interface) bool { return i.Name == named }) + if index < 0 { + return machine.Interface{}, fmt.Errorf("this machine has no interface %q, it has:\n%s", named, describeInterfaces(inventory.Interfaces)) + } + + return inventory.Interfaces[index], nil + } + + if index := interfaceOfHost(inventory.Interfaces, host); index >= 0 { + return inventory.Interfaces[index], nil + } + + options := make([]string, 0, len(inventory.Interfaces)) + for _, iface := range inventory.Interfaces { + options = append(options, describeInterface(iface)) + } + + title := fmt.Sprintf("%s is not an address of any interface this machine reports:", host) + + index, err := p.Choose(title, options, prompt.NoDefault) + if err != nil { + return machine.Interface{}, fmt.Errorf("%w. Name the interface with --network-interface", err) + } + + return inventory.Interfaces[index], nil +} + +func interfaceOfHost(interfaces []machine.Interface, host string) int { + for i, iface := range interfaces { + for _, address := range iface.Addresses { + if addressOf(address) == host { + return i + } + } + } + + return -1 +} + +// addressOf drops the prefix length an inventory address carries. +func addressOf(address string) string { + bare, _, _ := strings.Cut(address, "/") + + return bare +} + +// BuildDocument fills the machine's half into the cluster's template. The +// result is the document the machine boots from, and it carries a live +// bootstrap token: it is built in memory and never written anywhere. +func BuildDocument(template *unstructured.Unstructured, choices Choices) ([]byte, error) { + spec, found, err := unstructured.NestedMap(template.Object, "spec") + if err != nil || !found { + return nil, fmt.Errorf("the node configuration template of this group carries no spec: %w", err) + } + + document := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": documentAPIVersion, + "kind": documentKind, + "metadata": map[string]any{"name": choices.NodeName}, + "spec": spec, + }} + + if err := unstructured.SetNestedField(document.Object, choices.NodeName, "spec", "nodeName"); err != nil { + return nil, fmt.Errorf("set the node name: %w", err) + } + + if err := unstructured.SetNestedMap(document.Object, storageOf(choices), "spec", "storage"); err != nil { + return nil, fmt.Errorf("set the storage: %w", err) + } + + if err := unstructured.SetNestedMap(document.Object, networkOf(choices), "spec", "network"); err != nil { + return nil, fmt.Errorf("set the network: %w", err) + } + + body, err := yaml.Marshal(document.Object) + if err != nil { + return nil, fmt.Errorf("render the node configuration: %w", err) + } + + return body, nil +} + +func storageOf(choices Choices) map[string]any { + selector := map[string]any{} + for key, value := range choices.Selector { + selector[key] = value + } + + return map[string]any{"diskSelector": selector, "wipe": choices.Wipe} +} + +func networkOf(choices Choices) map[string]any { + iface := map[string]any{"name": choices.Interface.Name, "dhcp": !choices.StaticAddress} + + if choices.StaticAddress { + addresses := make([]any, 0, len(choices.Interface.Addresses)) + for _, address := range choices.Interface.Addresses { + addresses = append(addresses, address) + } + + iface["addresses"] = addresses + + if choices.Interface.Gateway != "" { + iface["gateway"] = choices.Interface.Gateway + } + } + + return map[string]any{"interfaces": []any{iface}} +} + +// Redact blanks the three fields a template carries live, so a document may be +// shown without handing over the right to add a node to the cluster. +func Redact(document []byte) ([]byte, error) { + object := map[string]any{} + if err := yaml.Unmarshal(document, &object); err != nil { + return nil, fmt.Errorf("read the document to redact it: %w", err) + } + + for _, path := range redactedPaths { + if _, found, _ := unstructured.NestedString(object, path...); !found { + continue + } + + if err := unstructured.SetNestedField(object, Redacted, path...); err != nil { + return nil, fmt.Errorf("redact %s: %w", strings.Join(path, "."), err) + } + } + + body, err := yaml.Marshal(object) + if err != nil { + return nil, fmt.Errorf("render the redacted document: %w", err) + } + + return body, nil +} + +func describeDisks(disks []machine.Disk) string { + lines := make([]string, 0, len(disks)) + for _, disk := range disks { + lines = append(lines, " "+describeDisk(disk)) + } + + return strings.Join(lines, "\n") +} + +func describeDisk(disk machine.Disk) string { + line := fmt.Sprintf("%-10s %8s %-14s %s", disk.Name, HumanSize(disk.Size), disk.State, strings.TrimSpace(disk.Vendor+" "+disk.Model)) + + if disk.Serial != "" { + line += fmt.Sprintf(" (serial %s)", disk.Serial) + } + + if disk.State == machine.StateSystemLayout { + line += " [holds an OS]" + } + + return strings.TrimSpace(line) +} + +func describeInterfaces(interfaces []machine.Interface) string { + lines := make([]string, 0, len(interfaces)) + for _, iface := range interfaces { + lines = append(lines, " "+describeInterface(iface)) + } + + return strings.Join(lines, "\n") +} + +func describeInterface(iface machine.Interface) string { + addresses := strings.Join(iface.Addresses, ", ") + if addresses == "" { + addresses = "no address" + } + + line := fmt.Sprintf("%-8s %-18s %-5s %s", iface.Name, iface.MAC, iface.Link, addresses) + + if iface.Gateway != "" { + line += " gw " + iface.Gateway + } + + return strings.TrimSpace(line) +} + +func firstNonEmptySerial(disks []machine.Disk) string { + for _, disk := range disks { + if disk.Serial != "" { + return disk.Serial + } + } + + return "" +} + +// HumanSize spells a byte count the way an operator reads a disk size. +func HumanSize(size uint64) string { + const unit = 1024 + + if size < unit { + return fmt.Sprintf("%dB", size) + } + + value, exponent := float64(size), 0 + for value >= unit && exponent < 5 { + value /= unit + exponent++ + } + + return fmt.Sprintf("%.0f%ci", value, "KMGTP"[exponent-1]) +} diff --git a/internal/olcedar/plan/plan_test.go b/internal/olcedar/plan/plan_test.go new file mode 100644 index 000000000..a299aad89 --- /dev/null +++ b/internal/olcedar/plan/plan_test.go @@ -0,0 +1,254 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + "github.com/deckhouse/deckhouse-cli/internal/olcedar/machine" + "github.com/deckhouse/deckhouse-cli/internal/olcedar/prompt" +) + +// bootstrapToken is the fixture secret every leak test looks for by value. +const bootstrapToken = "abcdef.0123456789abcdef" + +const registryAuth = "ZGVja2hvdXNlOnBhc3N3b3Jk" + +const proxyToken = "cmVnaXN0cnktcGFja2FnZXMtcHJveHktdG9rZW4=" + +func inventory() *machine.Inventory { + return &machine.Inventory{ + Disks: []machine.Disk{ + {Name: "nvme0n1", Size: 512110190592, Model: "MZVL2512", Serial: "S3Z8NB0K700002", State: machine.StateBlank}, + {Name: "sda", Size: 2000398934016, Model: "ST2000DM008", Serial: "ZA20ABCD", State: machine.StateSystemLayout}, + }, + Interfaces: []machine.Interface{ + {Name: "eno1", MAC: "aa:bb:cc:dd:ee:01", Link: "up", Addresses: []string{"10.12.4.55/24"}, Gateway: "10.12.4.1"}, + {Name: "eno2", MAC: "aa:bb:cc:dd:ee:02", Link: "down"}, + }, + } +} + +func template() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "templates.internal.deckhouse.io/v1alpha1", + "kind": "NodeConfigTemplate", + "metadata": map[string]any{"name": "worker"}, + "spec": map[string]any{ + "nodeName": "", + "osImage": map[string]any{"name": "olcedar"}, + "kubelet": map[string]any{"bootstrapToken": bootstrapToken, "maxPods": int64(120)}, + "registry": map[string]any{"auth": registryAuth}, + "registryPackagesProxyAccessTokenB64": proxyToken, + "network": map[string]any{}, + "storage": map[string]any{}, + }, + }} +} + +func asking(answers string) (*prompt.Prompt, *bytes.Buffer) { + out := &bytes.Buffer{} + + return prompt.New(strings.NewReader(answers), out, false), out +} + +func assuming() *prompt.Prompt { + return prompt.New(strings.NewReader(""), &bytes.Buffer{}, true) +} + +// A disk that already holds a system is never picked for the operator: the +// install erases it. +func TestChooseDiskRefusesSystemLayoutWithoutConfirmation(t *testing.T) { + _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) + require.ErrorContains(t, err, machine.StateSystemLayout) + require.ErrorContains(t, err, "--wipe") +} + +func TestChooseDiskAsksBeforeErasing(t *testing.T) { + p, out := asking("n\n") + + _, _, err := ChooseDisk(p, inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) + require.ErrorContains(t, err, "--wipe") + require.Contains(t, out.String(), "will erase it") + + p, _ = asking("y\n") + + disk, wipe, err := ChooseDisk(p, inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) + require.NoError(t, err) + require.Equal(t, "sda", disk.Name) + require.True(t, wipe) +} + +func TestChooseDiskTakesWipeAsTheConfirmation(t *testing.T) { + disk, wipe, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "ZA20ABCD"}, true) + require.NoError(t, err) + require.Equal(t, "sda", disk.Name) + require.True(t, wipe) +} + +// The single blank disk is the default, and it is still confirmed by pressing +// enter on a list that shows what else the machine has. +func TestChooseDiskDefaultsToTheOnlyBlankDisk(t *testing.T) { + p, out := asking("\n") + + disk, wipe, err := ChooseDisk(p, inventory(), nil, false) + require.NoError(t, err) + require.Equal(t, "nvme0n1", disk.Name) + require.False(t, wipe) + require.Contains(t, out.String(), "477Gi") + require.Contains(t, out.String(), "holds an OS") +} + +func TestChooseDiskRefusesToGuessBetweenTwoBlankDisks(t *testing.T) { + twoBlank := inventory() + twoBlank.Disks[1].State = machine.StateBlank + + _, _, err := ChooseDisk(assuming(), twoBlank, nil, false) + require.ErrorIs(t, err, prompt.ErrNoDefault) + require.ErrorContains(t, err, "--disk-selector") +} + +func TestChooseDiskRefusesASelectorMatchingSeveralDisks(t *testing.T) { + _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"name": "*"}, false) + require.ErrorContains(t, err, "matches 2 disks") +} + +func TestChooseDiskRefusesASelectorMatchingNothing(t *testing.T) { + _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "nosuchserial"}, false) + require.ErrorContains(t, err, "matches no disk") +} + +// The interface the CLI reached the machine on is proven by the connection, so +// it is taken without a question. +func TestChooseInterfaceTakesTheOneTheAddressIsOn(t *testing.T) { + iface, err := ChooseInterface(assuming(), inventory(), "10.12.4.55", "") + require.NoError(t, err) + require.Equal(t, "eno1", iface.Name) +} + +func TestChooseInterfaceAsksWhenTheAddressIsOnNone(t *testing.T) { + _, err := ChooseInterface(assuming(), inventory(), "192.0.2.10", "") + require.ErrorIs(t, err, prompt.ErrNoDefault) + require.ErrorContains(t, err, "--network-interface") + + p, out := asking("2\n") + + iface, err := ChooseInterface(p, inventory(), "192.0.2.10", "") + require.NoError(t, err) + require.Equal(t, "eno2", iface.Name) + require.Contains(t, out.String(), "is not an address of any interface") +} + +func TestChooseInterfaceRefusesAnInterfaceTheMachineLacks(t *testing.T) { + _, err := ChooseInterface(assuming(), inventory(), "10.12.4.55", "eth9") + require.ErrorContains(t, err, `no interface "eth9"`) +} + +func TestBuildDocumentFillsTheMachineHalfIntoTheClusterHalf(t *testing.T) { + document, err := BuildDocument(template(), Choices{ + NodeName: "worker-1", + Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, + Wipe: true, + Interface: inventory().Interfaces[0], + }) + require.NoError(t, err) + + object := map[string]any{} + require.NoError(t, yaml.Unmarshal(document, &object)) + + require.Equal(t, "internal.deckhouse.io/v1alpha1", object["apiVersion"]) + require.Equal(t, "NodeConfig", object["kind"]) + + name, _, _ := unstructured.NestedString(object, "metadata", "name") + require.Equal(t, "worker-1", name) + + nodeName, _, _ := unstructured.NestedString(object, "spec", "nodeName") + require.Equal(t, "worker-1", nodeName) + + serial, _, _ := unstructured.NestedString(object, "spec", "storage", "diskSelector", "serial") + require.Equal(t, "S3Z8NB0K700002", serial) + + wipe, _, _ := unstructured.NestedBool(object, "spec", "storage", "wipe") + require.True(t, wipe) + + interfaces, _, _ := unstructured.NestedSlice(object, "spec", "network", "interfaces") + require.Len(t, interfaces, 1) + require.Equal(t, "eno1", interfaces[0].(map[string]any)["name"]) + require.Equal(t, true, interfaces[0].(map[string]any)["dhcp"]) + + // The cluster's half has to survive: without the token kubelet has nothing + // to present on first contact. + token, _, _ := unstructured.NestedString(object, "spec", "kubelet", "bootstrapToken") + require.Equal(t, bootstrapToken, token) +} + +func TestBuildDocumentPinsTheAddressWhenAskedTo(t *testing.T) { + document, err := BuildDocument(template(), Choices{ + NodeName: "worker-1", + Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, + Interface: inventory().Interfaces[0], + StaticAddress: true, + }) + require.NoError(t, err) + + object := map[string]any{} + require.NoError(t, yaml.Unmarshal(document, &object)) + + interfaces, _, _ := unstructured.NestedSlice(object, "spec", "network", "interfaces") + iface := interfaces[0].(map[string]any) + require.Equal(t, false, iface["dhcp"]) + require.Equal(t, []any{"10.12.4.55/24"}, iface["addresses"]) + require.Equal(t, "10.12.4.1", iface["gateway"]) +} + +// Reading a template is the right to add a node to the cluster: the three live +// secrets it carries never reach an output stream. +func TestRedactHidesEverySecretTheTemplateCarries(t *testing.T) { + document, err := BuildDocument(template(), Choices{ + NodeName: "worker-1", + Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, + Interface: inventory().Interfaces[0], + }) + require.NoError(t, err) + require.Contains(t, string(document), bootstrapToken) + + redacted, err := Redact(document) + require.NoError(t, err) + + require.NotContains(t, string(redacted), bootstrapToken) + require.NotContains(t, string(redacted), registryAuth) + require.NotContains(t, string(redacted), proxyToken) + require.Equal(t, 3, strings.Count(string(redacted), Redacted)) + + // What is not a secret still has to be readable, or the redaction hid the + // document rather than its secrets. + require.Contains(t, string(redacted), "worker-1") + require.Contains(t, string(redacted), "S3Z8NB0K700002") + require.Contains(t, string(redacted), "maxPods") +} + +func TestHumanSize(t *testing.T) { + require.Equal(t, "477Gi", HumanSize(512110190592)) + require.Equal(t, "2Ti", HumanSize(2000398934016)) + require.Equal(t, "512B", HumanSize(512)) +} diff --git a/internal/olcedar/prompt/prompt.go b/internal/olcedar/prompt/prompt.go new file mode 100644 index 000000000..3c251e39c --- /dev/null +++ b/internal/olcedar/prompt/prompt.go @@ -0,0 +1,180 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package prompt asks the questions a machine cannot answer for itself. Every +// question reads from an io.Reader and writes to an io.Writer, so the whole +// command is testable without a terminal. +package prompt + +import ( + "bufio" + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +// ErrNoDefault means an answer was assumed but there is nothing to assume: the +// caller names the flag that would carry it. +var ErrNoDefault = errors.New("there is no default to assume") + +// NoDefault is the index Choose takes when no option may be picked silently. +const NoDefault = -1 + +// Prompt asks on a stream. Assume answers every question with its default +// instead of asking, which is what --yes means. +type Prompt struct { + in *bufio.Reader + out io.Writer + assume bool +} + +func New(in io.Reader, out io.Writer, assume bool) *Prompt { + return &Prompt{in: bufio.NewReader(in), out: out, assume: assume} +} + +// Printf writes to the same stream the questions go to. +func (p *Prompt) Printf(format string, args ...any) { + fmt.Fprintf(p.out, format, args...) +} + +// Choose shows a numbered list and returns the index picked. defaultIndex of +// NoDefault means the answer has to be typed: nothing here may be picked for +// the operator. +func (p *Prompt) Choose(title string, options []string, defaultIndex int) (int, error) { + if len(options) == 0 { + return 0, fmt.Errorf("%s: there is nothing to choose from", title) + } + + if p.assume { + if defaultIndex == NoDefault { + return 0, fmt.Errorf("%s: %w", title, ErrNoDefault) + } + + return defaultIndex, nil + } + + fmt.Fprintf(p.out, "\n%s\n\n", title) + + for i, option := range options { + fmt.Fprintf(p.out, " %d) %s\n", i+1, option) + } + + for { + answer, err := p.ask(question("Choice", defaultLabel(defaultIndex))) + if err != nil { + return 0, err + } + + if answer == "" && defaultIndex != NoDefault { + return defaultIndex, nil + } + + number, err := strconv.Atoi(answer) + if err != nil || number < 1 || number > len(options) { + fmt.Fprintf(p.out, "Answer with a number between 1 and %d.\n", len(options)) + + continue + } + + return number - 1, nil + } +} + +// Confirm asks a yes/no question. defaultYes decides what an empty line means, +// and a question that may destroy data is asked with defaultYes false. +func (p *Prompt) Confirm(text string, defaultYes bool) (bool, error) { + if p.assume { + return defaultYes, nil + } + + suffix := "y/N" + if defaultYes { + suffix = "Y/n" + } + + for { + answer, err := p.ask(fmt.Sprintf("%s [%s]: ", text, suffix)) + if err != nil { + return false, err + } + + switch strings.ToLower(answer) { + case "": + return defaultYes, nil + case "y", "yes": + return true, nil + case "n", "no": + return false, nil + default: + fmt.Fprintln(p.out, "Answer y or n.") + } + } +} + +// Line reads one line of text, falling back to defaultValue on an empty answer. +func (p *Prompt) Line(text, defaultValue string) (string, error) { + if p.assume { + if defaultValue == "" { + return "", fmt.Errorf("%s: %w", text, ErrNoDefault) + } + + return defaultValue, nil + } + + answer, err := p.ask(question(text, defaultValue)) + if err != nil { + return "", err + } + + if answer == "" { + return defaultValue, nil + } + + return answer, nil +} + +func (p *Prompt) ask(text string) (string, error) { + fmt.Fprint(p.out, text) + + answer, err := p.in.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", fmt.Errorf("read the answer: %w", err) + } + + if errors.Is(err, io.EOF) && strings.TrimSpace(answer) == "" { + return "", errors.New("read the answer: the input ended") + } + + return strings.TrimSpace(answer), nil +} + +func question(text, defaultValue string) string { + if defaultValue == "" { + return text + ": " + } + + return fmt.Sprintf("%s [%s]: ", text, defaultValue) +} + +func defaultLabel(defaultIndex int) string { + if defaultIndex == NoDefault { + return "" + } + + return strconv.Itoa(defaultIndex + 1) +} diff --git a/internal/olcedar/prompt/prompt_test.go b/internal/olcedar/prompt/prompt_test.go new file mode 100644 index 000000000..efa1bef52 --- /dev/null +++ b/internal/olcedar/prompt/prompt_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prompt + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func asking(answers string) (*Prompt, *bytes.Buffer) { + out := &bytes.Buffer{} + + return New(strings.NewReader(answers), out, false), out +} + +func TestChooseTakesTheNumberTyped(t *testing.T) { + p, out := asking("2\n") + + index, err := p.Choose("Disks:", []string{"nvme0n1", "sda"}, 0) + require.NoError(t, err) + require.Equal(t, 1, index) + require.Contains(t, out.String(), "1) nvme0n1") + require.Contains(t, out.String(), "2) sda") +} + +func TestChooseAsksAgainAfterAnAnswerOutOfRange(t *testing.T) { + p, out := asking("7\nx\n1\n") + + index, err := p.Choose("Disks:", []string{"nvme0n1", "sda"}, NoDefault) + require.NoError(t, err) + require.Equal(t, 0, index) + require.Equal(t, 2, strings.Count(out.String(), "Answer with a number between 1 and 2.")) +} + +func TestChooseTakesTheDefaultOnAnEmptyLine(t *testing.T) { + p, _ := asking("\n") + + index, err := p.Choose("Disks:", []string{"nvme0n1", "sda"}, 1) + require.NoError(t, err) + require.Equal(t, 1, index) +} + +// Assuming an answer is only allowed where there is one to assume. +func TestChooseRefusesToAssumeWithoutADefault(t *testing.T) { + p := New(strings.NewReader(""), &bytes.Buffer{}, true) + + _, err := p.Choose("Disks:", []string{"nvme0n1", "sda"}, NoDefault) + require.ErrorIs(t, err, ErrNoDefault) + + index, err := p.Choose("Disks:", []string{"nvme0n1", "sda"}, 1) + require.NoError(t, err) + require.Equal(t, 1, index) +} + +func TestConfirm(t *testing.T) { + p, _ := asking("y\n") + answer, err := p.Confirm("Erase it?", false) + require.NoError(t, err) + require.True(t, answer) + + p, _ = asking("\n") + answer, err = p.Confirm("Erase it?", false) + require.NoError(t, err) + require.False(t, answer) + + p, out := asking("maybe\nno\n") + answer, err = p.Confirm("Erase it?", true) + require.NoError(t, err) + require.False(t, answer) + require.Contains(t, out.String(), "Answer y or n.") +} + +func TestConfirmAssumesItsDefault(t *testing.T) { + p := New(strings.NewReader(""), &bytes.Buffer{}, true) + + answer, err := p.Confirm("Erase it?", false) + require.NoError(t, err) + require.False(t, answer) +} + +func TestLineFallsBackToTheDefault(t *testing.T) { + p, out := asking("\n") + + answer, err := p.Line("Node name", "worker-1") + require.NoError(t, err) + require.Equal(t, "worker-1", answer) + require.Contains(t, out.String(), "Node name [worker-1]: ") + + p, _ = asking("worker-7\n") + answer, err = p.Line("Node name", "worker-1") + require.NoError(t, err) + require.Equal(t, "worker-7", answer) +} + +func TestAskReportsInputThatEnded(t *testing.T) { + p, _ := asking("") + + _, err := p.Line("Node name", "") + require.ErrorContains(t, err, "the input ended") +} diff --git a/internal/system/cmd/system.go b/internal/system/cmd/system.go index 1447afacc..2f7a55dd8 100644 --- a/internal/system/cmd/system.go +++ b/internal/system/cmd/system.go @@ -23,6 +23,8 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/system/cmd/edit" "github.com/deckhouse/deckhouse-cli/internal/system/cmd/get" "github.com/deckhouse/deckhouse-cli/internal/system/cmd/logs" + + olcedar "github.com/deckhouse/deckhouse-cli/internal/olcedar/cmd" module "github.com/deckhouse/deckhouse-cli/internal/system/cmd/module/cmd" pkg "github.com/deckhouse/deckhouse-cli/internal/system/cmd/package/cmd" queue "github.com/deckhouse/deckhouse-cli/internal/system/cmd/queue" @@ -53,6 +55,7 @@ func NewCommand() *cobra.Command { collectdebuginfo.NewCommand(), queue.NewCommand(), logs.NewCommand(), + olcedar.NewCommand(), ) flags.AddPersistentFlags(systemCmd) From 11ca2e328711980d70a5ba2f9c612a15694da12e Mon Sep 17 00:00:00 2001 From: borg-z Date: Fri, 21 Aug 2026 21:28:43 +0300 Subject: [PATCH 2/4] [olcedar] Adopt a disk that already carries the layout instead of erasing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cut read `system-layout` as "an OS is here, the install will erase it" and wrote `spec.storage.wipe: true` once the operator confirmed. On a machine whose disk was cloned from an image — every DVP node — that is not a confirmation, it is a self-destruct. init decides for itself whether anything is installed: `resolveSystemDisk` (images/init/src/0.1/disk.go) finds one disk carrying BOOT/CONFIG/DATA and answers `install=false`, and initmode.go computes `provision := install || cfg.Spec.Storage.Wipe`. So `wipe: true` is the one thing that can force a provision where none is needed — and the provision is destructive first and sourced second: `provisionDisk` erases, and only then `copy-boot-assets` looks for the UKI and the rootfs under /run/media. A machine booted from its own disk has no such media, so the copy fails after the erase and there is nothing left to boot from. So: - a disk that already carries the layout is adopted, not installed onto, and the document names no storage at all: init identifies that disk itself, and a selector disagreeing with the pin recorded at install (writeDiskPin) is itself a reason for it to reinstall; - `wipe` reaches the document only from an explicit --wipe, which now reads as "reinstall, from installation media", and says what it costs without it; - a second disk carrying the layout is called out: the node identifies its disk by that layout before it reads any selector, so it would take that one and leave the named disk alone. What the notes warn about cannot be settled from an inventory: `diskState` spells installer media and an installed system the same way (inventory.go:101), and only the machine knows which disk it booted from. Adding that fact to the inventory belongs to the initramfs repository, where bootedDisk() already computes it. Co-Authored-By: Claude Opus 5 Signed-off-by: borg-z --- internal/olcedar/cmd/add/add.go | 6 +- internal/olcedar/plan/plan.go | 88 ++++++++++++++++------ internal/olcedar/plan/plan_test.go | 115 +++++++++++++++++++++-------- 3 files changed, 153 insertions(+), 56 deletions(-) diff --git a/internal/olcedar/cmd/add/add.go b/internal/olcedar/cmd/add/add.go index 8f97f5279..d28d65338 100644 --- a/internal/olcedar/cmd/add/add.go +++ b/internal/olcedar/cmd/add/add.go @@ -95,7 +95,7 @@ func NewCommand() *cobra.Command { flags.StringVar(&opts.diskSelector, "disk-selector", "", "Disk to install onto, as key=value (serial, wwid, name, busPath, model)") flags.StringVar(&opts.network, "network", "", "Network configuration: dhcp or static (default asks, dhcp)") flags.StringVar(&opts.networkInterface, "network-interface", "", "Interface to configure, when the address reaches the machine through a forward") - flags.BoolVar(&opts.wipe, "wipe", false, "Allow the install to erase a disk that is not blank") + flags.BoolVar(&opts.wipe, "wipe", false, "Erase the disk and reinstall onto it; only for a machine booted from installation media") flags.BoolVar(&opts.yes, "yes", false, "Answer every question with its default, and refuse where there is no default") flags.BoolVar(&opts.dryRun, "dry-run", false, "Print the document with its secrets redacted instead of pushing it") flags.BoolVar(&opts.wait, "wait", true, "Wait for the node to register in the cluster") @@ -269,7 +269,7 @@ func decide( return nil, err } - disk, wipe, err := plan.ChooseDisk(p, inventory, selector, opts.wipe) + disk, err := plan.ChooseDisk(p, inventory, selector, opts.wipe) if err != nil { return nil, err } @@ -293,7 +293,7 @@ func decide( NodeName: name, Disk: disk, Selector: machine.SelectorFor(disk), - Wipe: wipe, + Wipe: opts.wipe, Interface: iface, StaticAddress: static, }, nil diff --git a/internal/olcedar/plan/plan.go b/internal/olcedar/plan/plan.go index e3cfc9e16..010be7e6b 100644 --- a/internal/olcedar/plan/plan.go +++ b/internal/olcedar/plan/plan.go @@ -62,40 +62,69 @@ type Choices struct { StaticAddress bool } -// ChooseDisk settles which disk the OS is installed onto, and whether the -// install may erase what is on it. A selector given on the command line has to -// resolve to exactly one disk; without one the operator picks from the list. -func ChooseDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine.Selector, wipe bool) (machine.Disk, bool, error) { +// ChooseDisk settles which disk the node runs on. A disk that already carries +// BOOT/CONFIG/DATA is adopted rather than installed onto: init identifies it +// itself and provisions nothing. Mirrors resolveSystemDisk of +// images/init/src/0.1/disk.go in the initramfs repository, where an installed +// disk yields install=false. +func ChooseDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine.Selector, wipe bool) (machine.Disk, error) { if len(inventory.Disks) == 0 { - return machine.Disk{}, false, errors.New("this machine reports no disks, so there is nothing to install onto") + return machine.Disk{}, errors.New("this machine reports no disks, so there is nothing to install onto") } disk, err := pickDisk(p, inventory, selector) if err != nil { - return machine.Disk{}, false, err + return machine.Disk{}, err } - if disk.State == machine.StateBlank { - return disk, false, nil + for _, note := range DiskNotes(inventory, disk, wipe) { + p.Printf("%s\n", note) } + return disk, nil +} + +// DiskNotes says what this disk means for the machine, in the order an operator +// needs to hear it. Nothing here refuses: what the notes warn about cannot be +// settled from an inventory, only by the operator who knows how the machine +// booted. +func DiskNotes(inventory *machine.Inventory, disk machine.Disk, wipe bool) []string { + var notes []string + if wipe { - return disk, true, nil + notes = append(notes, fmt.Sprintf( + "--wipe erases %s and reinstalls onto it. The install copies the UKI and the rootfs from "+ + "the installer media under /run/media, and a machine booted from its own disk has none: "+ + "there the erase succeeds and the copy then fails, leaving nothing to boot from. "+ + "Pass it only for a machine booted from installation media.", disk.Name)) + } else if disk.State != machine.StateBlank { + notes = append(notes, fmt.Sprintf( + "%s already carries this system's layout, so it is adopted as it is: nothing is installed and "+ + "nothing is erased. Reinstalling onto it takes --wipe, and only from installation media.", disk.Name)) } - confirmed, err := p.Confirm(fmt.Sprintf( - "Disk %s is %s and the install will erase it. Continue?", disk.Name, disk.State), false) - if err != nil { - return machine.Disk{}, false, err + if other, found := otherLayoutDisk(inventory, disk); found { + notes = append(notes, fmt.Sprintf( + "%s also carries this system's layout. The node identifies its disk by that layout before it reads "+ + "any selector, so it would take %s and leave %s alone. Detach %s before adding this machine.", + other.Name, other.Name, disk.Name, other.Name)) } - if !confirmed { - return machine.Disk{}, false, fmt.Errorf( - "disk %s is %s, not %s: erasing it has to be confirmed, or allowed with --wipe", - disk.Name, disk.State, machine.StateBlank) + return notes +} + +// otherLayoutDisk is a disk that is not the chosen one and still carries the +// layout, which the node would pick over what this document names. +func otherLayoutDisk(inventory *machine.Inventory, chosen machine.Disk) (machine.Disk, bool) { + for _, disk := range inventory.Disks { + if disk.Name == chosen.Name || disk.State != machine.StateSystemLayout { + continue + } + + return disk, true } - return disk, true, nil + return machine.Disk{}, false } func pickDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine.Selector) (machine.Disk, error) { @@ -230,8 +259,10 @@ func BuildDocument(template *unstructured.Unstructured, choices Choices) ([]byte return nil, fmt.Errorf("set the node name: %w", err) } - if err := unstructured.SetNestedMap(document.Object, storageOf(choices), "spec", "storage"); err != nil { - return nil, fmt.Errorf("set the storage: %w", err) + if storage, install := storageOf(choices); install { + if err := unstructured.SetNestedMap(document.Object, storage, "spec", "storage"); err != nil { + return nil, fmt.Errorf("set the storage: %w", err) + } } if err := unstructured.SetNestedMap(document.Object, networkOf(choices), "spec", "network"); err != nil { @@ -246,13 +277,26 @@ func BuildDocument(template *unstructured.Unstructured, choices Choices) ([]byte return body, nil } -func storageOf(choices Choices) map[string]any { +// storageOf names the disk to install onto. A machine whose disk already carries +// the layout is handed no storage at all: the node identifies that disk itself, +// and a selector that disagrees with the pin recorded at install would send it +// through a reinstall it has no media for. +func storageOf(choices Choices) (map[string]any, bool) { + if !choices.Wipe && choices.Disk.State != machine.StateBlank { + return nil, false + } + selector := map[string]any{} for key, value := range choices.Selector { selector[key] = value } - return map[string]any{"diskSelector": selector, "wipe": choices.Wipe} + storage := map[string]any{"diskSelector": selector} + if choices.Wipe { + storage["wipe"] = true + } + + return storage, true } func networkOf(choices Choices) map[string]any { diff --git a/internal/olcedar/plan/plan_test.go b/internal/olcedar/plan/plan_test.go index a299aad89..9c9380164 100644 --- a/internal/olcedar/plan/plan_test.go +++ b/internal/olcedar/plan/plan_test.go @@ -76,34 +76,40 @@ func assuming() *prompt.Prompt { return prompt.New(strings.NewReader(""), &bytes.Buffer{}, true) } -// A disk that already holds a system is never picked for the operator: the -// install erases it. -func TestChooseDiskRefusesSystemLayoutWithoutConfirmation(t *testing.T) { - _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) - require.ErrorContains(t, err, machine.StateSystemLayout) - require.ErrorContains(t, err, "--wipe") -} - -func TestChooseDiskAsksBeforeErasing(t *testing.T) { - p, out := asking("n\n") - - _, _, err := ChooseDisk(p, inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) - require.ErrorContains(t, err, "--wipe") - require.Contains(t, out.String(), "will erase it") +// A disk that already carries the layout is adopted, not installed onto: init +// identifies it itself and provisions nothing. Erasing it is a separate, +// explicit decision. +func TestChooseDiskAdoptsADiskThatCarriesTheLayout(t *testing.T) { + p, out := asking("") - p, _ = asking("y\n") - - disk, wipe, err := ChooseDisk(p, inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) + disk, err := ChooseDisk(p, inventory(), machine.Selector{"serial": "ZA20ABCD"}, false) require.NoError(t, err) require.Equal(t, "sda", disk.Name) - require.True(t, wipe) + require.Contains(t, out.String(), "adopted as it is") + require.Contains(t, out.String(), "--wipe") } -func TestChooseDiskTakesWipeAsTheConfirmation(t *testing.T) { - disk, wipe, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "ZA20ABCD"}, true) - require.NoError(t, err) - require.Equal(t, "sda", disk.Name) - require.True(t, wipe) +// --wipe is the reinstall, and a reinstall needs installation media: init +// erases the disk first and only then looks for the UKI under /run/media. +func TestDiskNotesWarnThatWipeNeedsInstallationMedia(t *testing.T) { + notes := DiskNotes(inventory(), inventory().Disks[1], true) + require.Len(t, notes, 1) + require.Contains(t, notes[0], "/run/media") + require.Contains(t, notes[0], "nothing to boot from") +} + +// The node identifies its disk by the layout before it reads any selector, so a +// second disk carrying one silently wins over what the document names. +func TestDiskNotesWarnAboutAnotherDiskCarryingTheLayout(t *testing.T) { + notes := DiskNotes(inventory(), inventory().Disks[0], false) + require.Len(t, notes, 1) + require.Contains(t, notes[0], "sda also carries") + require.Contains(t, notes[0], "Detach sda") +} + +func TestDiskNotesAreSilentOnASingleBlankDisk(t *testing.T) { + oneDisk := &machine.Inventory{Disks: []machine.Disk{{Name: "nvme0n1", State: machine.StateBlank}}} + require.Empty(t, DiskNotes(oneDisk, oneDisk.Disks[0], false)) } // The single blank disk is the default, and it is still confirmed by pressing @@ -111,10 +117,9 @@ func TestChooseDiskTakesWipeAsTheConfirmation(t *testing.T) { func TestChooseDiskDefaultsToTheOnlyBlankDisk(t *testing.T) { p, out := asking("\n") - disk, wipe, err := ChooseDisk(p, inventory(), nil, false) + disk, err := ChooseDisk(p, inventory(), nil, false) require.NoError(t, err) require.Equal(t, "nvme0n1", disk.Name) - require.False(t, wipe) require.Contains(t, out.String(), "477Gi") require.Contains(t, out.String(), "holds an OS") } @@ -123,18 +128,18 @@ func TestChooseDiskRefusesToGuessBetweenTwoBlankDisks(t *testing.T) { twoBlank := inventory() twoBlank.Disks[1].State = machine.StateBlank - _, _, err := ChooseDisk(assuming(), twoBlank, nil, false) + _, err := ChooseDisk(assuming(), twoBlank, nil, false) require.ErrorIs(t, err, prompt.ErrNoDefault) require.ErrorContains(t, err, "--disk-selector") } func TestChooseDiskRefusesASelectorMatchingSeveralDisks(t *testing.T) { - _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"name": "*"}, false) + _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"name": "*"}, false) require.ErrorContains(t, err, "matches 2 disks") } func TestChooseDiskRefusesASelectorMatchingNothing(t *testing.T) { - _, _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "nosuchserial"}, false) + _, err := ChooseDisk(assuming(), inventory(), machine.Selector{"serial": "nosuchserial"}, false) require.ErrorContains(t, err, "matches no disk") } @@ -167,8 +172,8 @@ func TestChooseInterfaceRefusesAnInterfaceTheMachineLacks(t *testing.T) { func TestBuildDocumentFillsTheMachineHalfIntoTheClusterHalf(t *testing.T) { document, err := BuildDocument(template(), Choices{ NodeName: "worker-1", + Disk: inventory().Disks[0], Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, - Wipe: true, Interface: inventory().Interfaces[0], }) require.NoError(t, err) @@ -188,8 +193,11 @@ func TestBuildDocumentFillsTheMachineHalfIntoTheClusterHalf(t *testing.T) { serial, _, _ := unstructured.NestedString(object, "spec", "storage", "diskSelector", "serial") require.Equal(t, "S3Z8NB0K700002", serial) - wipe, _, _ := unstructured.NestedBool(object, "spec", "storage", "wipe") - require.True(t, wipe) + // Nothing asked for an erase, so nothing in the document asks for one: with + // wipe unset the node installs onto a blank disk and provisions nothing on + // a disk that already carries the layout. + _, found, _ := unstructured.NestedBool(object, "spec", "storage", "wipe") + require.False(t, found) interfaces, _, _ := unstructured.NestedSlice(object, "spec", "network", "interfaces") require.Len(t, interfaces, 1) @@ -202,9 +210,53 @@ func TestBuildDocumentFillsTheMachineHalfIntoTheClusterHalf(t *testing.T) { require.Equal(t, bootstrapToken, token) } +// A machine whose disk already carries the layout is handed no storage at all: +// the node identifies that disk itself, and a selector disagreeing with the pin +// recorded at install would send it through a reinstall it has no media for. +func TestBuildDocumentOmitsStorageForAnAdoptedDisk(t *testing.T) { + document, err := BuildDocument(template(), Choices{ + NodeName: "worker-1", + Disk: inventory().Disks[1], + Selector: machine.Selector{"serial": "ZA20ABCD"}, + Interface: inventory().Interfaces[0], + }) + require.NoError(t, err) + + object := map[string]any{} + require.NoError(t, yaml.Unmarshal(document, &object)) + + _, found, _ := unstructured.NestedString(object, "spec", "storage", "diskSelector", "serial") + require.False(t, found) + + _, found, _ = unstructured.NestedBool(object, "spec", "storage", "wipe") + require.False(t, found) +} + +// --wipe is the only thing that writes wipe, and it names the disk to erase. +func TestBuildDocumentWritesWipeOnlyWhenAskedTo(t *testing.T) { + document, err := BuildDocument(template(), Choices{ + NodeName: "worker-1", + Disk: inventory().Disks[1], + Selector: machine.Selector{"serial": "ZA20ABCD"}, + Wipe: true, + Interface: inventory().Interfaces[0], + }) + require.NoError(t, err) + + object := map[string]any{} + require.NoError(t, yaml.Unmarshal(document, &object)) + + wipe, _, _ := unstructured.NestedBool(object, "spec", "storage", "wipe") + require.True(t, wipe) + + serial, _, _ := unstructured.NestedString(object, "spec", "storage", "diskSelector", "serial") + require.Equal(t, "ZA20ABCD", serial) +} + func TestBuildDocumentPinsTheAddressWhenAskedTo(t *testing.T) { document, err := BuildDocument(template(), Choices{ NodeName: "worker-1", + Disk: inventory().Disks[0], Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, Interface: inventory().Interfaces[0], StaticAddress: true, @@ -226,6 +278,7 @@ func TestBuildDocumentPinsTheAddressWhenAskedTo(t *testing.T) { func TestRedactHidesEverySecretTheTemplateCarries(t *testing.T) { document, err := BuildDocument(template(), Choices{ NodeName: "worker-1", + Disk: inventory().Disks[0], Selector: machine.Selector{"serial": "S3Z8NB0K700002"}, Interface: inventory().Interfaces[0], }) From 9665ad5f7eee3d665a1a0672562d7dc180f40f53 Mon Sep 17 00:00:00 2001 From: borg-z Date: Fri, 21 Aug 2026 21:34:49 +0300 Subject: [PATCH 3/4] [olcedar] Space the questions apart, and report where the time went MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run on zykov-st read as one paragraph: the answer to a question, the note about the disk and the next question all ran together, and the wait that followed said nothing for minutes. So the prompt sets its blocks apart itself — a blank line after a list, before a question, and around a note — rather than leaving each caller to remember a newline. A machine with one disk now offers it as the default: there is no choice to make, and the list still shows what the disk is. The wait says it is still waiting every 30 seconds, and the command closes with what the addition cost: Timings read the cluster and the machine 1.2s pushed the configuration 0.3s node registered 1m24s machine time in total 1m24s The operator's own thinking time sits between the read and the push and is deliberately not counted: the number worth knowing is how long the machine takes, not how long someone stared at a disk list. Co-Authored-By: Claude Opus 5 Signed-off-by: borg-z --- internal/olcedar/cluster/cluster.go | 41 +++++++++++++--- internal/olcedar/cluster/cluster_test.go | 17 +++++++ internal/olcedar/cmd/add/add.go | 60 ++++++++++++++++++++---- internal/olcedar/plan/plan.go | 6 ++- internal/olcedar/prompt/prompt.go | 12 +++++ 5 files changed, 120 insertions(+), 16 deletions(-) diff --git a/internal/olcedar/cluster/cluster.go b/internal/olcedar/cluster/cluster.go index a1fa3d317..9d1ac78fa 100644 --- a/internal/olcedar/cluster/cluster.go +++ b/internal/olcedar/cluster/cluster.go @@ -195,15 +195,42 @@ func NodeExists(ctx context.Context, kube kubernetes.Interface, name string) (bo return true, nil } -// WaitForNode waits until the node registers. Registration is what this command -// can promise: it means the machine installed itself and its kubelet reached -// the cluster. Readiness comes later, from the modules rolled onto the node. -func WaitForNode(ctx context.Context, kube kubernetes.Interface, name string, timeout time.Duration) error { - err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { - return NodeExists(ctx, kube, name) +// pollInterval is how often the cluster is asked about the node, and +// tickInterval how often the wait says out loud that it is still waiting. +const ( + pollInterval = 5 * time.Second + tickInterval = 30 * time.Second +) + +// WaitForNode waits until the node registers, reporting how long it has been +// waiting through tick. Registration is what this command can promise: it means +// the machine installed itself and its kubelet reached the cluster. Readiness +// comes later, from the modules rolled onto the node. +func WaitForNode( + ctx context.Context, + kube kubernetes.Interface, + name string, + timeout time.Duration, + tick func(elapsed time.Duration), +) error { + started := time.Now() + ticked := started + + err := wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, func(ctx context.Context) (bool, error) { + registered, err := NodeExists(ctx, kube, name) + if err != nil || registered { + return registered, err + } + + if time.Since(ticked) >= tickInterval { + ticked = time.Now() + tick(time.Since(started).Round(time.Second)) + } + + return false, nil }) if err != nil { - return fmt.Errorf("wait for node %s to register: %w", name, err) + return fmt.Errorf("wait for node %s to register after %s: %w", name, time.Since(started).Round(time.Second), err) } return nil diff --git a/internal/olcedar/cluster/cluster_test.go b/internal/olcedar/cluster/cluster_test.go index 89772f632..595eba98b 100644 --- a/internal/olcedar/cluster/cluster_test.go +++ b/internal/olcedar/cluster/cluster_test.go @@ -19,6 +19,7 @@ package cluster import ( "context" "testing" + "time" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -134,6 +135,22 @@ func TestNodeNameByAddressNamesTheNodeHoldingTheAddress(t *testing.T) { require.Empty(t, NodeNameByAddress(context.Background(), kube, "10.12.4.56")) } +// The wait says out loud that it is still waiting, and reports how long it +// waited when the node never registers. +func TestWaitForNodeReportsHowLongItWaited(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := WaitForNode(ctx, fake.NewSimpleClientset(), "worker-1", 10*time.Millisecond, func(time.Duration) {}) + require.ErrorContains(t, err, "wait for node worker-1 to register after") +} + +func TestWaitForNodeReturnsAsSoonAsTheNodeIsThere(t *testing.T) { + err := WaitForNode(context.Background(), fake.NewSimpleClientset(node("worker-1", "worker")), + "worker-1", time.Minute, func(time.Duration) {}) + require.NoError(t, err) +} + func TestNodeExists(t *testing.T) { kube := fake.NewSimpleClientset(node("worker-0", "worker")) diff --git a/internal/olcedar/cmd/add/add.go b/internal/olcedar/cmd/add/add.go index d28d65338..8ccb26847 100644 --- a/internal/olcedar/cmd/add/add.go +++ b/internal/olcedar/cmd/add/add.go @@ -137,6 +137,7 @@ func run(cmd *cobra.Command, target string, opts *options) error { ctx := cmd.Context() address := machine.Address(target) p := prompt.New(cmd.InOrStdin(), cmd.OutOrStdout(), opts.yes) + started := time.Now() kube, dyn, err := clients(cmd) if err != nil { @@ -157,11 +158,15 @@ func run(cmd *cobra.Command, target string, opts *options) error { return err } + read := time.Since(started) + choices, err := decide(ctx, p, kube, inventory, machine.Host(address), opts) if err != nil { return err } + decided := time.Now() + document, err := plan.BuildDocument(template, *choices) if err != nil { return err @@ -179,13 +184,40 @@ func run(cmd *cobra.Command, target string, opts *options) error { return err } - p.Printf("\n%s accepted the configuration and is installing itself as %s.\n", address, choices.NodeName) + push := time.Since(decided) + + p.Printf("\n%s took the configuration and is bringing itself up as %s.\n", address, choices.NodeName) if !opts.wait { + printTimings(p, read, push, 0) + return nil } - return waitForNode(ctx, p, kube, choices.NodeName, opts.waitTimeout) + registration, err := waitForNode(ctx, p, kube, choices.NodeName, opts.waitTimeout) + if err != nil { + return err + } + + printTimings(p, read, push, registration) + + return nil +} + +// printTimings says where the time went, so the cost of adding a node is a +// measurement rather than an impression. The operator's own thinking time sits +// between the read and the push and is deliberately not counted. +func printTimings(p *prompt.Prompt, read, push, registration time.Duration) { + p.Printf("\nTimings\n") + p.Printf(" read the cluster and the machine %8s\n", read.Round(time.Millisecond*100)) + p.Printf(" pushed the configuration %8s\n", push.Round(time.Millisecond*100)) + + if registration == 0 { + return + } + + p.Printf(" node registered %8s\n", registration.Round(time.Second)) + p.Printf(" machine time in total %8s\n", (push + registration).Round(time.Second)) } func validate(opts *options, interactive bool) error { @@ -366,17 +398,29 @@ func printRedacted(cmd *cobra.Command, document []byte) error { return nil } -func waitForNode(ctx context.Context, p *prompt.Prompt, kube kubernetes.Interface, name string, timeout time.Duration) error { +func waitForNode( + ctx context.Context, + p *prompt.Prompt, + kube kubernetes.Interface, + name string, + timeout time.Duration, +) (time.Duration, error) { started := time.Now() p.Printf("Waiting for %s to register, up to %s.\n", name, timeout) - if err := cluster.WaitForNode(ctx, kube, name, timeout); err != nil { - return err + tick := func(elapsed time.Duration) { + p.Printf(" still waiting, %s\n", elapsed) + } + + if err := cluster.WaitForNode(ctx, kube, name, timeout, tick); err != nil { + return 0, err } - p.Printf("%s registered in %s. It becomes Ready once the cluster rolls its modules onto it:\n d8 k wait --for=condition=Ready node/%s\n", - name, time.Since(started).Round(time.Second), name) + registration := time.Since(started) - return nil + p.Printf("\n%s registered in %s. It turns Ready once the cluster rolls its modules onto it:\n d8 k wait --for=condition=Ready node/%s\n", + name, registration.Round(time.Second), name) + + return registration, nil } diff --git a/internal/olcedar/plan/plan.go b/internal/olcedar/plan/plan.go index 010be7e6b..6944b6c64 100644 --- a/internal/olcedar/plan/plan.go +++ b/internal/olcedar/plan/plan.go @@ -78,7 +78,7 @@ func ChooseDisk(p *prompt.Prompt, inventory *machine.Inventory, selector machine } for _, note := range DiskNotes(inventory, disk, wipe) { - p.Printf("%s\n", note) + p.Note(note) } return disk, nil @@ -166,6 +166,10 @@ func matchOne(selector machine.Selector, disks []machine.Disk) (machine.Disk, er // defaultDisk offers the one blank disk, and nothing when there is a choice to // make: a disk holding data is never picked for the operator. func defaultDisk(disks []machine.Disk) int { + if len(disks) == 1 { + return 0 + } + found := prompt.NoDefault for i, disk := range disks { diff --git a/internal/olcedar/prompt/prompt.go b/internal/olcedar/prompt/prompt.go index 3c251e39c..cdb4272b7 100644 --- a/internal/olcedar/prompt/prompt.go +++ b/internal/olcedar/prompt/prompt.go @@ -52,6 +52,12 @@ func (p *Prompt) Printf(format string, args ...any) { fmt.Fprintf(p.out, format, args...) } +// Note writes a paragraph the operator has to read, set off by a blank line so +// it does not run into the answer above it or the question below. +func (p *Prompt) Note(text string) { + fmt.Fprintf(p.out, "\n%s\n", text) +} + // Choose shows a numbered list and returns the index picked. defaultIndex of // NoDefault means the answer has to be typed: nothing here may be picked for // the operator. @@ -74,6 +80,8 @@ func (p *Prompt) Choose(title string, options []string, defaultIndex int) (int, fmt.Fprintf(p.out, " %d) %s\n", i+1, option) } + fmt.Fprintln(p.out) + for { answer, err := p.ask(question("Choice", defaultLabel(defaultIndex))) if err != nil { @@ -107,6 +115,8 @@ func (p *Prompt) Confirm(text string, defaultYes bool) (bool, error) { suffix = "Y/n" } + fmt.Fprintln(p.out) + for { answer, err := p.ask(fmt.Sprintf("%s [%s]: ", text, suffix)) if err != nil { @@ -136,6 +146,8 @@ func (p *Prompt) Line(text, defaultValue string) (string, error) { return defaultValue, nil } + fmt.Fprintln(p.out) + answer, err := p.ask(question(text, defaultValue)) if err != nil { return "", err From dc258dc932425314e20da043c6efe2ce324426ec Mon Sep 17 00:00:00 2001 From: borg-z Date: Fri, 21 Aug 2026 21:42:55 +0300 Subject: [PATCH 4/4] [olcedar] Bound the cluster reads, and say which one is being waited on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run against 10.12.0.40 hung with nothing on the screen at all. The machine was fine — it had already answered /whoami — and the command was sitting in the template read: nodeconfigtemplates is served by an aggregated API, so the kube-apiserver proxies it to node-controller, and client-go carries no timeout of its own. A backend that never answers hangs the command forever. So the rest config now bounds every cluster read at 30 seconds, and each network step says what it is about to do before it does it. Two silent minutes become two lines and, at worst, one bounded failure. That failure names where the answer comes from, because the kube-apiserver is almost never the part that is broken: read the node configuration template of worker: . It is served by an aggregated API, so this read is proxied by the kube-apiserver to node-controller of node-manager: a node-controller that is down, unreachable or unregistered fails it. Check it with: d8 k get apiservice v1alpha1.templates.internal.deckhouse.io Co-Authored-By: Claude Opus 5 Signed-off-by: borg-z --- internal/olcedar/cluster/cluster.go | 5 ++++- internal/olcedar/cluster/cluster_test.go | 16 ++++++++++++++++ internal/olcedar/cmd/add/add.go | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/internal/olcedar/cluster/cluster.go b/internal/olcedar/cluster/cluster.go index 9d1ac78fa..e9f298c3a 100644 --- a/internal/olcedar/cluster/cluster.go +++ b/internal/olcedar/cluster/cluster.go @@ -72,7 +72,10 @@ func FetchTemplate(ctx context.Context, dyn dynamic.Interface, group string) (*u } if !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("read the node configuration template of %s: %w", group, err) + return nil, fmt.Errorf("read the node configuration template of %s: %w. "+ + "It is served by an aggregated API, so this read is proxied by the kube-apiserver to node-controller "+ + "of node-manager: a node-controller that is down, unreachable or unregistered fails it. "+ + "Check it with: d8 k get apiservice v1alpha1.templates.internal.deckhouse.io", group, err) } return nil, explainMissingTemplate(ctx, dyn, group) diff --git a/internal/olcedar/cluster/cluster_test.go b/internal/olcedar/cluster/cluster_test.go index 595eba98b..8c1c5927e 100644 --- a/internal/olcedar/cluster/cluster_test.go +++ b/internal/olcedar/cluster/cluster_test.go @@ -23,12 +23,14 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) func node(name, group string, addresses ...string) *corev1.Node { @@ -86,6 +88,20 @@ func TestFetchTemplateExplainsAGroupOfTheWrongType(t *testing.T) { require.ErrorContains(t, err, "provisioned by the cluster itself") } +// A read that fails for any reason other than a missing template names where +// the answer comes from: the failure is almost never in the kube-apiserver. +func TestFetchTemplateNamesTheAggregatedAPIOnFailure(t *testing.T) { + client := dynamicClient() + client.PrependReactor("get", "nodeconfigtemplates", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewTimeoutError("the backend did not answer", 30) + }) + + _, err := FetchTemplate(context.Background(), client, "worker") + require.ErrorContains(t, err, "aggregated API") + require.ErrorContains(t, err, "node-controller") + require.ErrorContains(t, err, "d8 k get apiservice") +} + func TestFetchTemplateExplainsAMissingGroup(t *testing.T) { _, err := FetchTemplate(context.Background(), dynamicClient(), "worker") require.ErrorContains(t, err, `there is no NodeGroup "worker"`) diff --git a/internal/olcedar/cmd/add/add.go b/internal/olcedar/cmd/add/add.go index 8ccb26847..c83eeff88 100644 --- a/internal/olcedar/cmd/add/add.go +++ b/internal/olcedar/cmd/add/add.go @@ -43,6 +43,12 @@ const ( networkStatic = "static" ) +// clusterRequestTimeout bounds one read of the cluster. The template is served +// by an aggregated API: the kube-apiserver proxies the request to +// node-controller, and a backend that does not answer would otherwise hang the +// command with nothing on the screen. +const clusterRequestTimeout = 30 * time.Second + var addLong = templates.LongDesc(` Add a machine to the cluster as a static node running an immutable OS (olcedar). @@ -144,15 +150,21 @@ func run(cmd *cobra.Command, target string, opts *options) error { return err } + p.Printf("asking %s who holds the maintenance port\n", address) + if err := checkMachineIsNotANode(ctx, kube, address); err != nil { return err } + p.Printf("reading the node configuration template of %s\n", opts.group) + template, err := cluster.FetchTemplate(ctx, dyn, opts.group) if err != nil { return err } + p.Printf("reading the inventory of %s\n", address) + inventory, err := machine.FetchInventory(ctx, address) if err != nil { return err @@ -252,6 +264,13 @@ func clients(cmd *cobra.Command) (kubernetes.Interface, dynamic.Interface, error return nil, nil, fmt.Errorf("set up the Kubernetes client: %w", err) } + restConfig.Timeout = clusterRequestTimeout + + kube, err = kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, nil, fmt.Errorf("set up the Kubernetes client: %w", err) + } + dyn, err := dynamic.NewForConfig(restConfig) if err != nil { return nil, nil, fmt.Errorf("set up the dynamic Kubernetes client: %w", err)