diff --git a/pkg/coordinator/tasks/check_consensus_config_spec/README.md b/pkg/coordinator/tasks/check_consensus_config_spec/README.md new file mode 100644 index 00000000..2bd510fb --- /dev/null +++ b/pkg/coordinator/tasks/check_consensus_config_spec/README.md @@ -0,0 +1,131 @@ +# Check Consensus Config Spec Task + +This task validates that consensus clients properly implement the `/eth/v1/config/spec` endpoint according to the Ethereum consensus specification. + +## Description + +The task fetches the latest consensus specification from the Ethereum consensus-specs repository, combining the main config with all relevant preset files (phase0, altair, bellatrix, capella, deneb, electra, fulu, etc.), and compares it against the response from each consensus client's `/eth/v1/config/spec` endpoint. It checks for: + +1. **Missing required fields** - Fields that should be present according to the spec but are missing from the client response +2. **Extra fields** - Fields in the client response that are not in the specification (can be allowed or disallowed) +3. **Value mismatches** - Fields where the client returns a different value than expected + +## Configuration + +```yaml +- name: check_consensus_config_spec + title: "Validate consensus config spec endpoint" + config: + clientPattern: ".*" # Regex pattern to filter clients + pollInterval: 10s # How often to check clients + networkPreset: "mainnet" # Network preset (mainnet or minimal only) + specBranch: "dev" # Git branch to use (dev, master, etc.) + presetFiles: [] # Optional: override default preset files + requiredFields: [] # If empty, uses all fields from combined spec + allowExtraFields: true # Whether to allow extra fields not in spec +``` + +## Parameters + +- `clientPattern`: Regex pattern to filter which clients to check (default: ".*" - all clients) +- `pollInterval`: Interval between checks (default: 10s) +- `networkPreset`: Network preset to use (default: "mainnet", can be "mainnet" or "minimal" only) +- `specBranch`: Git branch to use for fetching specs (default: "dev") +- `presetFiles`: List of preset files to fetch and combine (default: ["phase0.yaml", "altair.yaml", "bellatrix.yaml", "capella.yaml", "deneb.yaml", "electra.yaml", "fulu.yaml"]) +- `requiredFields`: List of fields that must be present (default: empty, uses all fields from combined spec) +- `allowExtraFields`: Whether to allow fields not in the specification (default: true) + +**Note**: The task automatically constructs the correct URLs based on `networkPreset` and `specBranch`: +- Config: `https://raw.githubusercontent.com/ethereum/consensus-specs/{specBranch}/configs/{networkPreset}.yaml` +- Presets: + - For `minimal`: `https://raw.githubusercontent.com/ethereum/consensus-specs/{specBranch}/presets/minimal/{presetFile}` + - For `mainnet`: `https://raw.githubusercontent.com/ethereum/consensus-specs/{specBranch}/presets/mainnet/{presetFile}` + +## Outputs + +The task sets the following output variables: + +- `validationSummary`: Object containing: + - `totalClients`: Total number of clients checked + - `validClients`: Number of clients that passed validation + - `invalidClients`: Number of clients that failed validation + - `results`: Array of validation results for each client + +Each result in the `results` array contains: +- `name`: Client name +- `isValid`: Whether the client passed validation +- `missingFields`: Array of required fields that are missing +- `extraFields`: Array of extra fields (if not allowed) +- `errorMessage`: Error message if the client couldn't be checked +- `receivedSpec`: The actual spec returned by the client +- `comparisonIssues`: Array of value mismatches found + +## Example Usage + +### Basic usage - check all clients (defaults to mainnet) +```yaml +- name: check_consensus_config_spec + title: "Validate all clients against mainnet spec" + config: + clientPattern: ".*" +``` + +### Check specific clients only +```yaml +- name: check_consensus_config_spec + title: "Validate Lighthouse clients only" + config: + clientPattern: "lighthouse-.*" +``` + +### Use minimal preset +```yaml +- name: check_consensus_config_spec + title: "Validate against minimal preset" + config: + networkPreset: "minimal" +``` + +### Use different network config (holesky example with mainnet presets) +```yaml +- name: check_consensus_config_spec + title: "Validate against holesky config with mainnet presets" + config: + # Note: Only mainnet or minimal are valid for networkPreset + # For other networks like holesky, they use mainnet presets + networkPreset: "mainnet" +``` + +### Use master branch instead of dev +```yaml +- name: check_consensus_config_spec + title: "Validate against master branch specs" + config: + specBranch: "master" +``` + +### Strict validation (no extra fields allowed) +```yaml +- name: check_consensus_config_spec + title: "Strict spec validation" + config: + allowExtraFields: false +``` + +## Task Behavior + +- The task starts by fetching the expected specification from the configured source +- It then continuously polls all matching clients to check their spec endpoint +- For each client, it validates the response against the expected specification +- Results are logged with clear indication of which clients are missing which fields +- The task succeeds only if all clients pass validation + +## Error Handling + +The task handles various error scenarios: +- Network errors when fetching the specification +- Client timeout or connection errors +- Invalid response format from clients +- Missing or malformed data in client responses + +All errors are logged with appropriate context and client identification. \ No newline at end of file diff --git a/pkg/coordinator/tasks/check_consensus_config_spec/config.go b/pkg/coordinator/tasks/check_consensus_config_spec/config.go new file mode 100644 index 00000000..80593047 --- /dev/null +++ b/pkg/coordinator/tasks/check_consensus_config_spec/config.go @@ -0,0 +1,77 @@ +package checkconsensusconfigspec + +import ( + "fmt" + "time" + + "github.com/ethpandaops/assertoor/pkg/coordinator/helper" +) + +type Config struct { + // ClientPattern is a regex pattern to filter clients + ClientPattern string `yaml:"clientPattern" json:"clientPattern"` + // PollInterval is the interval to poll for client updates + PollInterval helper.Duration `yaml:"pollInterval" json:"pollInterval"` + // NetworkPreset is the network preset to use (mainnet or minimal only) + NetworkPreset string `yaml:"networkPreset" json:"networkPreset"` + // SpecBranch is the git branch to use for fetching specs (defaults to "dev") + SpecBranch string `yaml:"specBranch" json:"specBranch"` + // PresetFiles is the list of preset files to fetch and combine (optional override) + PresetFiles []string `yaml:"presetFiles" json:"presetFiles"` + // RequiredFields specifies which fields are mandatory in the spec response + RequiredFields []string `yaml:"requiredFields" json:"requiredFields"` + // AllowExtraFields determines if extra fields not in the spec are allowed + AllowExtraFields bool `yaml:"allowExtraFields" json:"allowExtraFields"` + + // Internal computed fields (not configurable by user) + specSource string + presetBaseURL string +} + +func DefaultConfig() Config { + return Config{ + ClientPattern: ".*", + PollInterval: helper.Duration{Duration: 10 * time.Second}, + NetworkPreset: "mainnet", + SpecBranch: "dev", + PresetFiles: []string{"phase0.yaml", "altair.yaml", "bellatrix.yaml", "capella.yaml", "deneb.yaml", "electra.yaml", "fulu.yaml"}, + RequiredFields: []string{}, // Will be populated from combined spec + AllowExtraFields: true, + } +} + +func (c *Config) Validate() error { + if c.ClientPattern == "" { + c.ClientPattern = ".*" + } + if c.PollInterval.Duration == 0 { + c.PollInterval.Duration = 10 * time.Second + } + if c.NetworkPreset == "" { + c.NetworkPreset = "mainnet" + } + if c.SpecBranch == "" { + c.SpecBranch = "dev" + } + if len(c.PresetFiles) == 0 { + c.PresetFiles = []string{"phase0.yaml", "altair.yaml", "bellatrix.yaml", "capella.yaml", "deneb.yaml", "electra.yaml", "fulu.yaml"} + } + + // Validate networkPreset - only mainnet or minimal are valid + if c.NetworkPreset != "mainnet" && c.NetworkPreset != "minimal" { + return fmt.Errorf("invalid networkPreset '%s': only 'mainnet' or 'minimal' are supported", c.NetworkPreset) + } + + // Compute the URLs based on the network preset and branch + c.specSource = fmt.Sprintf("https://raw.githubusercontent.com/ethereum/consensus-specs/%s/configs/%s.yaml", c.SpecBranch, c.NetworkPreset) + + // For minimal preset, use the correct path + if c.NetworkPreset == "minimal" { + c.presetBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/ethereum/consensus-specs/%s/presets/minimal", c.SpecBranch) + } else { + // For mainnet, use the standard mainnet presets + c.presetBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/ethereum/consensus-specs/%s/presets/mainnet", c.SpecBranch) + } + + return nil +} \ No newline at end of file diff --git a/pkg/coordinator/tasks/check_consensus_config_spec/task.go b/pkg/coordinator/tasks/check_consensus_config_spec/task.go new file mode 100644 index 00000000..cc985933 --- /dev/null +++ b/pkg/coordinator/tasks/check_consensus_config_spec/task.go @@ -0,0 +1,343 @@ +package checkconsensusconfigspec + +import ( + "context" + "fmt" + "io" + "net/http" + "reflect" + "strings" + "time" + + "github.com/ethpandaops/assertoor/pkg/coordinator/clients" + "github.com/ethpandaops/assertoor/pkg/coordinator/types" + "github.com/ethpandaops/assertoor/pkg/coordinator/vars" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +var ( + TaskName = "check_consensus_config_spec" + TaskDescriptor = &types.TaskDescriptor{ + Name: TaskName, + Description: "Checks consensus clients for compliance with the /eth/v1/config/spec endpoint specification.", + Config: DefaultConfig(), + NewTask: NewTask, + } +) + +type Task struct { + ctx *types.TaskContext + options *types.TaskOptions + config Config + logger logrus.FieldLogger + expectedSpec map[string]interface{} + expectedSpecKeys []string +} + +type ClientValidationResult struct { + Name string `json:"name"` + IsValid bool `json:"isValid"` + MissingFields []string `json:"missingFields"` + ExtraFields []string `json:"extraFields"` + ErrorMessage string `json:"errorMessage"` + ReceivedSpec map[string]interface{} `json:"receivedSpec"` + ComparisonIssues []string `json:"comparisonIssues"` +} + +type ValidationSummary struct { + TotalClients int `json:"totalClients"` + ValidClients int `json:"validClients"` + InvalidClients int `json:"invalidClients"` + Results []*ClientValidationResult `json:"results"` +} + +func NewTask(ctx *types.TaskContext, options *types.TaskOptions) (types.Task, error) { + return &Task{ + ctx: ctx, + options: options, + logger: ctx.Logger.GetLogger(), + }, nil +} + +func (t *Task) Config() interface{} { + return t.config +} + +func (t *Task) Timeout() time.Duration { + return t.options.Timeout.Duration +} + +func (t *Task) LoadConfig() error { + config := DefaultConfig() + + // parse static config + if t.options.Config != nil { + if err := t.options.Config.Unmarshal(&config); err != nil { + return fmt.Errorf("error parsing task config for %v: %w", TaskName, err) + } + } + + // load dynamic vars + err := t.ctx.Vars.ConsumeVars(&config, t.options.ConfigVars) + if err != nil { + return err + } + + // validate config + if err := config.Validate(); err != nil { + return err + } + + t.config = config + + return nil +} + +func (t *Task) Execute(ctx context.Context) error { + // Fetch expected spec from the source + if err := t.fetchExpectedSpec(ctx); err != nil { + return fmt.Errorf("failed to fetch expected spec: %w", err) + } + + t.logger.Infof("Loaded combined spec with %d fields from main config and %d preset files", + len(t.expectedSpecKeys), len(t.config.PresetFiles)) + + // Initial check + t.processCheck(ctx) + + // Poll for updates + for { + select { + case <-time.After(t.config.PollInterval.Duration): + t.processCheck(ctx) + case <-ctx.Done(): + return nil + } + } +} + +func (t *Task) fetchExpectedSpec(ctx context.Context) error { + // Initialize the combined spec + t.expectedSpec = make(map[string]interface{}) + + // First, fetch the main config spec + if err := t.fetchConfigFile(ctx, t.config.specSource); err != nil { + return fmt.Errorf("failed to fetch main config spec: %w", err) + } + + // Then fetch and combine all preset files + for _, presetFile := range t.config.PresetFiles { + presetURL := fmt.Sprintf("%s/%s", t.config.presetBaseURL, presetFile) + if err := t.fetchConfigFile(ctx, presetURL); err != nil { + // Log warning but continue if preset file is not found (some might be optional) + t.logger.Warnf("Failed to fetch preset file %s: %v", presetFile, err) + continue + } + } + + // Build the keys list from the combined spec + t.expectedSpecKeys = make([]string, 0, len(t.expectedSpec)) + for key := range t.expectedSpec { + t.expectedSpecKeys = append(t.expectedSpecKeys, key) + } + + // If no required fields specified in config, use all fields from combined spec + if len(t.config.RequiredFields) == 0 { + t.config.RequiredFields = t.expectedSpecKeys + } + + t.logger.Infof("Combined spec contains %d total fields from main config and %d preset files", + len(t.expectedSpecKeys), len(t.config.PresetFiles)) + + return nil +} + +func (t *Task) fetchConfigFile(ctx context.Context, url string) error { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return fmt.Errorf("failed to create request for %s: %w", url, err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to fetch %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to fetch %s, status code: %d", url, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body from %s: %w", url, err) + } + + var config map[string]interface{} + if err := yaml.Unmarshal(body, &config); err != nil { + return fmt.Errorf("failed to parse YAML from %s: %w", url, err) + } + + // Merge the config into the expected spec + for key, value := range config { + if existingValue, exists := t.expectedSpec[key]; exists { + t.logger.Debugf("Overriding spec field %s: %v -> %v", key, existingValue, value) + } + t.expectedSpec[key] = value + } + + return nil +} + +func (t *Task) processCheck(ctx context.Context) { + allResultsPass := true + validationSummary := &ValidationSummary{ + Results: []*ClientValidationResult{}, + } + + clients := t.ctx.Scheduler.GetServices().ClientPool().GetClientsByNamePatterns(t.config.ClientPattern, "") + validationSummary.TotalClients = len(clients) + + for _, client := range clients { + result := t.validateClient(ctx, client) + validationSummary.Results = append(validationSummary.Results, result) + + if result.IsValid { + validationSummary.ValidClients++ + } else { + validationSummary.InvalidClients++ + allResultsPass = false + } + } + + t.logValidationResults(validationSummary) + + // Set output variables + if validationData, err := vars.GeneralizeData(validationSummary); err == nil { + t.ctx.Outputs.SetVar("validationSummary", validationData) + } else { + t.logger.Warnf("Failed setting `validationSummary` output: %v", err) + } + + if allResultsPass { + t.ctx.SetResult(types.TaskResultSuccess) + } else { + t.ctx.SetResult(types.TaskResultNone) + } +} + +func (t *Task) validateClient(ctx context.Context, client *clients.PoolClient) *ClientValidationResult { + result := &ClientValidationResult{ + Name: client.Config.Name, + MissingFields: []string{}, + ExtraFields: []string{}, + ComparisonIssues: []string{}, + } + + checkLogger := t.logger.WithField("client", client.Config.Name) + + // Fetch spec from client + receivedSpec, err := client.ConsensusClient.GetRPCClient().GetConfigSpecs(ctx) + if err != nil { + checkLogger.Errorf("Failed to fetch config specs: %v", err) + result.ErrorMessage = fmt.Sprintf("Failed to fetch config specs: %v", err) + return result + } + + if receivedSpec == nil { + checkLogger.Error("Received nil config specs response") + result.ErrorMessage = "Received nil config specs response" + return result + } + + result.ReceivedSpec = receivedSpec + + // Check for missing required fields + for _, requiredField := range t.config.RequiredFields { + if _, exists := receivedSpec[requiredField]; !exists { + result.MissingFields = append(result.MissingFields, requiredField) + } + } + + // Check for extra fields if not allowed + if !t.config.AllowExtraFields { + expectedFieldsMap := make(map[string]bool) + for _, field := range t.expectedSpecKeys { + expectedFieldsMap[field] = true + } + + for receivedField := range receivedSpec { + if !expectedFieldsMap[receivedField] { + result.ExtraFields = append(result.ExtraFields, receivedField) + } + } + } + + // Compare field values with expected spec + for field, expectedValue := range t.expectedSpec { + if receivedValue, exists := receivedSpec[field]; exists { + if !t.compareValues(expectedValue, receivedValue) { + result.ComparisonIssues = append(result.ComparisonIssues, + fmt.Sprintf("Field '%s': expected %v (type: %T), got %v (type: %T)", + field, expectedValue, expectedValue, receivedValue, receivedValue)) + } + } + } + + result.IsValid = len(result.MissingFields) == 0 && + (t.config.AllowExtraFields || len(result.ExtraFields) == 0) && + len(result.ComparisonIssues) == 0 + + return result +} + +func (t *Task) compareValues(expected, received interface{}) bool { + // Convert both values to strings for comparison to handle type differences + expectedStr := fmt.Sprintf("%v", expected) + receivedStr := fmt.Sprintf("%v", received) + + // Try to compare as strings first + if expectedStr == receivedStr { + return true + } + + // For numeric values, try to compare with type conversion + if reflect.TypeOf(expected).Kind() == reflect.TypeOf(received).Kind() { + return reflect.DeepEqual(expected, received) + } + + return false +} + +func (t *Task) logValidationResults(summary *ValidationSummary) { + if summary.InvalidClients == 0 { + t.logger.Infof("✅ All %d clients passed spec validation", summary.TotalClients) + return + } + + t.logger.Errorf("❌ %d/%d clients failed spec validation", summary.InvalidClients, summary.TotalClients) + + for _, result := range summary.Results { + if !result.IsValid { + clientLogger := t.logger.WithField("client", result.Name) + + if result.ErrorMessage != "" { + clientLogger.Errorf("Error: %s", result.ErrorMessage) + continue + } + + if len(result.MissingFields) > 0 { + clientLogger.Errorf("Missing required fields: %s", strings.Join(result.MissingFields, ", ")) + } + + if len(result.ExtraFields) > 0 { + clientLogger.Warnf("Extra fields not in spec: %s", strings.Join(result.ExtraFields, ", ")) + } + + if len(result.ComparisonIssues) > 0 { + clientLogger.Errorf("Value mismatches: %s", strings.Join(result.ComparisonIssues, "; ")) + } + } + } +} \ No newline at end of file diff --git a/pkg/coordinator/tasks/tasks.go b/pkg/coordinator/tasks/tasks.go index bd482f49..2e4e769f 100644 --- a/pkg/coordinator/tasks/tasks.go +++ b/pkg/coordinator/tasks/tasks.go @@ -6,6 +6,7 @@ import ( checkclientsarehealthy "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_clients_are_healthy" checkconsensusattestationstats "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_attestation_stats" checkconsensusblockproposals "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_block_proposals" + checkconsensusconfigspec "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_config_spec" checkconsensusfinality "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_finality" checkconsensusforks "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_forks" checkconsensusproposerduty "github.com/ethpandaops/assertoor/pkg/coordinator/tasks/check_consensus_proposer_duty" @@ -45,6 +46,7 @@ var AvailableTaskDescriptors = []*types.TaskDescriptor{ checkclientsarehealthy.TaskDescriptor, checkconsensusattestationstats.TaskDescriptor, checkconsensusblockproposals.TaskDescriptor, + checkconsensusconfigspec.TaskDescriptor, checkconsensusfinality.TaskDescriptor, checkconsensusforks.TaskDescriptor, checkconsensusproposerduty.TaskDescriptor,