Skip to content

Commit 4d4872f

Browse files
edersonbrilhantegithub-actions[bot]npalm
authored
feat: support multiple SSM parameters for large runner matcher configs (#4790) (#4792)
### **PR Description** Implements support for splitting the runner matcher configuration across multiple SSM parameters. `PARAMETER_RUNNER_MATCHER_CONFIG_PATH` can now accept multiple parameter paths separated by a colon (`:`). This avoids SSM size limits for large configurations and improves scalability for environments with many runner types and labels. Closes #4790 --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Niek Palm <npalm@users.noreply.github.com>
1 parent 47b9e70 commit 4d4872f

File tree

10 files changed

+179
-25
lines changed

10 files changed

+179
-25
lines changed

‎lambdas/functions/webhook/src/ConfigLoader.test.ts‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,54 @@ describe('ConfigLoader Tests', () => {
168168
'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', // eslint-disable-line max-len
169169
);
170170
});
171+
172+
it('should load config successfully from multiple paths', async () => {
173+
process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
174+
process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
175+
176+
const partialMatcher1 =
177+
'[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}';
178+
const partialMatcher2 =
179+
',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]';
180+
181+
const combinedMatcherConfig = [
182+
{ id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } },
183+
{ id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } },
184+
];
185+
186+
vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
187+
if (paramPath === '/path/to/matcher/config-1') return partialMatcher1;
188+
if (paramPath === '/path/to/matcher/config-2') return partialMatcher2;
189+
if (paramPath === '/path/to/webhook/secret') return 'secret';
190+
return '';
191+
});
192+
193+
const config: ConfigWebhook = await ConfigWebhook.load();
194+
195+
expect(config.matcherConfig).toEqual(combinedMatcherConfig);
196+
expect(config.webhookSecret).toBe('secret');
197+
});
198+
199+
it('should throw error if config loading fails from multiple paths', async () => {
200+
process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
201+
process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
202+
203+
const partialMatcher1 =
204+
'[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}';
205+
const partialMatcher2 =
206+
',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}';
207+
208+
vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
209+
if (paramPath === '/path/to/matcher/config-1') return partialMatcher1;
210+
if (paramPath === '/path/to/matcher/config-2') return partialMatcher2;
211+
if (paramPath === '/path/to/webhook/secret') return 'secret';
212+
return '';
213+
});
214+
215+
await expect(ConfigWebhook.load()).rejects.toThrow(
216+
"Failed to load config: Failed to parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", // eslint-disable-line max-len
217+
);
218+
});
171219
});
172220

173221
describe('ConfigWebhookEventBridge', () => {
@@ -229,6 +277,32 @@ describe('ConfigLoader Tests', () => {
229277
expect(config.matcherConfig).toEqual(matcherConfig);
230278
});
231279

280+
it('should load config successfully from multiple paths with repo allow list', async () => {
281+
process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
282+
process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
283+
284+
const partial1 =
285+
'[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}';
286+
const partial2 =
287+
',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]';
288+
289+
const combined: RunnerMatcherConfig[] = [
290+
{ id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } },
291+
{ id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } },
292+
];
293+
294+
vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
295+
if (paramPath === '/path/to/matcher/config-1') return partial1;
296+
if (paramPath === '/path/to/matcher/config-2') return partial2;
297+
return '';
298+
});
299+
300+
const config: ConfigDispatcher = await ConfigDispatcher.load();
301+
302+
expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']);
303+
expect(config.matcherConfig).toEqual(combined);
304+
});
305+
232306
it('should throw error if config loading fails', async () => {
233307
vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
234308
throw new Error(`Parameter ${paramPath} not found`);
@@ -276,7 +350,7 @@ describe('ConfigLoader Tests', () => {
276350
return '';
277351
});
278352

279-
await expect(ConfigDispatcher.load()).rejects.toThrow('ailed to load config: Matcher config is empty');
353+
await expect(ConfigDispatcher.load()).rejects.toThrow('Failed to load config: Matcher config is empty');
280354
});
281355
});
282356
});

‎lambdas/functions/webhook/src/ConfigLoader.ts‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,17 +87,44 @@ abstract class BaseConfig {
8787
}
8888
}
8989

90-
export class ConfigWebhook extends BaseConfig {
91-
repositoryAllowList: string[] = [];
90+
abstract class MatcherAwareConfig extends BaseConfig {
9291
matcherConfig: RunnerMatcherConfig[] = [];
92+
93+
protected async loadMatcherConfig(paramPathsEnv: string) {
94+
if (!paramPathsEnv || paramPathsEnv === 'undefined' || paramPathsEnv === 'null' || !paramPathsEnv.includes(':')) {
95+
// Single path or invalid string → load directly
96+
await this.loadParameter(paramPathsEnv, 'matcherConfig');
97+
return;
98+
}
99+
100+
const paths = paramPathsEnv
101+
.split(':')
102+
.map((p) => p.trim())
103+
.filter(Boolean);
104+
let combinedString = '';
105+
for (const path of paths) {
106+
await this.loadParameter(path, 'matcherConfig');
107+
combinedString += this.matcherConfig;
108+
}
109+
110+
try {
111+
this.matcherConfig = JSON.parse(combinedString);
112+
} catch (error) {
113+
this.configLoadingErrors.push(`Failed to parse combined matcher config: ${(error as Error).message}`);
114+
}
115+
}
116+
}
117+
118+
export class ConfigWebhook extends MatcherAwareConfig {
119+
repositoryAllowList: string[] = [];
93120
webhookSecret: string = '';
94121
workflowJobEventSecondaryQueue: string = '';
95122

96123
async loadConfig(): Promise<void> {
97124
this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
98125

99126
await Promise.all([
100-
this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'),
127+
this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH),
101128
this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'),
102129
]);
103130

@@ -121,14 +148,13 @@ export class ConfigWebhookEventBridge extends BaseConfig {
121148
}
122149
}
123150

124-
export class ConfigDispatcher extends BaseConfig {
151+
export class ConfigDispatcher extends MatcherAwareConfig {
125152
repositoryAllowList: string[] = [];
126-
matcherConfig: RunnerMatcherConfig[] = [];
127153
workflowJobEventSecondaryQueue: string = ''; // Deprecated
128154

129155
async loadConfig(): Promise<void> {
130156
this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
131-
await this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig');
157+
await this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH);
132158

133159
validateRunnerMatcherConfig(this);
134160
}

‎modules/webhook/direct/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ No modules.
4040

4141
| Name | Description | Type | Default | Required |
4242
|------|-------------|------|---------|:--------:|
43-
| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/> prefix = string<br/> archive = optional(object({<br/> enable = optional(bool, true)<br/> retention_days = optional(number, 7)<br/> }), {})<br/> tags = optional(map(string), {})<br/><br/> lambda_subnet_ids = optional(list(string), [])<br/> lambda_security_group_ids = optional(list(string), [])<br/> sqs_job_queues_arns = list(string)<br/> lambda_zip = optional(string, null)<br/> lambda_memory_size = optional(number, 256)<br/> lambda_timeout = optional(number, 10)<br/> role_permissions_boundary = optional(string, null)<br/> role_path = optional(string, null)<br/> logging_retention_in_days = optional(number, 180)<br/> logging_kms_key_id = optional(string, null)<br/> lambda_s3_bucket = optional(string, null)<br/> lambda_s3_key = optional(string, null)<br/> lambda_s3_object_version = optional(string, null)<br/> lambda_apigateway_access_log_settings = optional(object({<br/> destination_arn = string<br/> format = string<br/> }), null)<br/> repository_white_list = optional(list(string), [])<br/> kms_key_arn = optional(string, null)<br/> log_level = optional(string, "info")<br/> lambda_runtime = optional(string, "nodejs22.x")<br/> aws_partition = optional(string, "aws")<br/> lambda_architecture = optional(string, "arm64")<br/> github_app_parameters = object({<br/> webhook_secret = map(string)<br/> })<br/> tracing_config = optional(object({<br/> mode = optional(string, null)<br/> capture_http_requests = optional(bool, false)<br/> capture_error = optional(bool, false)<br/> }), {})<br/> lambda_tags = optional(map(string), {})<br/> api_gw_source_arn = string<br/> ssm_parameter_runner_matcher_config = object({<br/> name = string<br/> arn = string<br/> version = string<br/> })<br/> })</pre> | n/a | yes |
43+
| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/> prefix = string<br/> archive = optional(object({<br/> enable = optional(bool, true)<br/> retention_days = optional(number, 7)<br/> }), {})<br/> tags = optional(map(string), {})<br/><br/> lambda_subnet_ids = optional(list(string), [])<br/> lambda_security_group_ids = optional(list(string), [])<br/> sqs_job_queues_arns = list(string)<br/> lambda_zip = optional(string, null)<br/> lambda_memory_size = optional(number, 256)<br/> lambda_timeout = optional(number, 10)<br/> role_permissions_boundary = optional(string, null)<br/> role_path = optional(string, null)<br/> logging_retention_in_days = optional(number, 180)<br/> logging_kms_key_id = optional(string, null)<br/> lambda_s3_bucket = optional(string, null)<br/> lambda_s3_key = optional(string, null)<br/> lambda_s3_object_version = optional(string, null)<br/> lambda_apigateway_access_log_settings = optional(object({<br/> destination_arn = string<br/> format = string<br/> }), null)<br/> repository_white_list = optional(list(string), [])<br/> kms_key_arn = optional(string, null)<br/> log_level = optional(string, "info")<br/> lambda_runtime = optional(string, "nodejs22.x")<br/> aws_partition = optional(string, "aws")<br/> lambda_architecture = optional(string, "arm64")<br/> github_app_parameters = object({<br/> webhook_secret = map(string)<br/> })<br/> tracing_config = optional(object({<br/> mode = optional(string, null)<br/> capture_http_requests = optional(bool, false)<br/> capture_error = optional(bool, false)<br/> }), {})<br/> lambda_tags = optional(map(string), {})<br/> api_gw_source_arn = string<br/> ssm_parameter_runner_matcher_config = list(object({<br/> name = string<br/> arn = string<br/> version = string<br/> }))<br/> })</pre> | n/a | yes |
4444

4545
## Outputs
4646

‎modules/webhook/direct/variables.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ variable "config" {
4141
}), {})
4242
lambda_tags = optional(map(string), {})
4343
api_gw_source_arn = string
44-
ssm_parameter_runner_matcher_config = object({
44+
ssm_parameter_runner_matcher_config = list(object({
4545
name = string
4646
arn = string
4747
version = string
48-
})
48+
}))
4949
})
5050
}

‎modules/webhook/direct/webhook.tf‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ resource "aws_lambda_function" "webhook" {
2626
POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error
2727
PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name
2828
REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list)
29-
PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name
30-
PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter
29+
PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
30+
PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter
3131
} : k => v if v != null
3232
}
3333
}
@@ -134,7 +134,12 @@ resource "aws_iam_role_policy" "webhook_ssm" {
134134
role = aws_iam_role.webhook_lambda.name
135135

136136
policy = templatefile("${path.module}/../policies/lambda-ssm.json", {
137-
resource_arns = jsonencode([var.config.github_app_parameters.webhook_secret.arn, var.config.ssm_parameter_runner_matcher_config.arn])
137+
resource_arns = jsonencode(
138+
concat(
139+
[var.config.github_app_parameters.webhook_secret.arn],
140+
[for p in var.config.ssm_parameter_runner_matcher_config : p.arn]
141+
)
142+
)
138143
})
139144
}
140145

‎modules/webhook/eventbridge/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ No modules.
5454

5555
| Name | Description | Type | Default | Required |
5656
|------|-------------|------|---------|:--------:|
57-
| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/> prefix = string<br/> archive = optional(object({<br/> enable = optional(bool, true)<br/> retention_days = optional(number, 7)<br/> }), {})<br/> tags = optional(map(string), {})<br/><br/> lambda_subnet_ids = optional(list(string), [])<br/> lambda_security_group_ids = optional(list(string), [])<br/> sqs_job_queues_arns = list(string)<br/> lambda_zip = optional(string, null)<br/> lambda_memory_size = optional(number, 256)<br/> lambda_timeout = optional(number, 10)<br/> role_permissions_boundary = optional(string, null)<br/> role_path = optional(string, null)<br/> logging_retention_in_days = optional(number, 180)<br/> logging_kms_key_id = optional(string, null)<br/> lambda_s3_bucket = optional(string, null)<br/> lambda_s3_key = optional(string, null)<br/> lambda_s3_object_version = optional(string, null)<br/> lambda_apigateway_access_log_settings = optional(object({<br/> destination_arn = string<br/> format = string<br/> }), null)<br/> repository_white_list = optional(list(string), [])<br/> kms_key_arn = optional(string, null)<br/> log_level = optional(string, "info")<br/> lambda_runtime = optional(string, "nodejs22.x")<br/> aws_partition = optional(string, "aws")<br/> lambda_architecture = optional(string, "arm64")<br/> github_app_parameters = object({<br/> webhook_secret = map(string)<br/> })<br/> tracing_config = optional(object({<br/> mode = optional(string, null)<br/> capture_http_requests = optional(bool, false)<br/> capture_error = optional(bool, false)<br/> }), {})<br/> lambda_tags = optional(map(string), {})<br/> api_gw_source_arn = string<br/> ssm_parameter_runner_matcher_config = object({<br/> name = string<br/> arn = string<br/> version = string<br/> })<br/> accept_events = optional(list(string), null)<br/> })</pre> | n/a | yes |
57+
| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/> prefix = string<br/> archive = optional(object({<br/> enable = optional(bool, true)<br/> retention_days = optional(number, 7)<br/> }), {})<br/> tags = optional(map(string), {})<br/><br/> lambda_subnet_ids = optional(list(string), [])<br/> lambda_security_group_ids = optional(list(string), [])<br/> sqs_job_queues_arns = list(string)<br/> lambda_zip = optional(string, null)<br/> lambda_memory_size = optional(number, 256)<br/> lambda_timeout = optional(number, 10)<br/> role_permissions_boundary = optional(string, null)<br/> role_path = optional(string, null)<br/> logging_retention_in_days = optional(number, 180)<br/> logging_kms_key_id = optional(string, null)<br/> lambda_s3_bucket = optional(string, null)<br/> lambda_s3_key = optional(string, null)<br/> lambda_s3_object_version = optional(string, null)<br/> lambda_apigateway_access_log_settings = optional(object({<br/> destination_arn = string<br/> format = string<br/> }), null)<br/> repository_white_list = optional(list(string), [])<br/> kms_key_arn = optional(string, null)<br/> log_level = optional(string, "info")<br/> lambda_runtime = optional(string, "nodejs22.x")<br/> aws_partition = optional(string, "aws")<br/> lambda_architecture = optional(string, "arm64")<br/> github_app_parameters = object({<br/> webhook_secret = map(string)<br/> })<br/> tracing_config = optional(object({<br/> mode = optional(string, null)<br/> capture_http_requests = optional(bool, false)<br/> capture_error = optional(bool, false)<br/> }), {})<br/> lambda_tags = optional(map(string), {})<br/> api_gw_source_arn = string<br/> ssm_parameter_runner_matcher_config = list(object({<br/> name = string<br/> arn = string<br/> version = string<br/> }))<br/> accept_events = optional(list(string), null)<br/> })</pre> | n/a | yes |
5858

5959
## Outputs
6060

‎modules/webhook/eventbridge/dispatcher.tf‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ resource "aws_lambda_function" "dispatcher" {
4444
POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.tracing_config.capture_http_requests
4545
POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error
4646
# Parameters required for lambda configuration
47-
PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name
48-
PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter
47+
PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
48+
PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter
4949
REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list)
5050
} : k => v if v != null
5151
}
@@ -129,7 +129,11 @@ resource "aws_iam_role_policy" "dispatcher_ssm" {
129129
role = aws_iam_role.dispatcher_lambda.name
130130

131131
policy = templatefile("${path.module}/../policies/lambda-ssm.json", {
132-
resource_arns = jsonencode([var.config.ssm_parameter_runner_matcher_config.arn])
132+
resource_arns = jsonencode(
133+
concat(
134+
[for p in var.config.ssm_parameter_runner_matcher_config : p.arn]
135+
)
136+
)
133137
})
134138
}
135139

‎modules/webhook/eventbridge/variables.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,11 @@ variable "config" {
4141
}), {})
4242
lambda_tags = optional(map(string), {})
4343
api_gw_source_arn = string
44-
ssm_parameter_runner_matcher_config = object({
44+
ssm_parameter_runner_matcher_config = list(object({
4545
name = string
4646
arn = string
4747
version = string
48-
})
48+
}))
4949
accept_events = optional(list(string), null)
5050
})
5151
}

0 commit comments

Comments
 (0)