();
+ const invalidMessages: string[] = [];
+ for (const payload of payloads) {
+ const { eventType, messageId, repositoryName, repositoryOwner } = payload;
+ if (ephemeralEnabled && eventType !== 'workflow_job') {
+ logger.warn(
+ 'Event is not supported in combination with ephemeral runners. Please ensure you have enabled workflow_job events.',
+ { eventType, messageId },
+ );
+
+ invalidMessages.push(messageId);
+
+ continue;
+ }
+
+ if (!isValidRepoOwnerTypeIfOrgLevelEnabled(payload, enableOrgLevel)) {
+ logger.warn(
+ `Repository does not belong to a GitHub organization and organization runners are enabled. This is not supported. Not scaling up for this event. Not throwing error to prevent re-queueing and just ignoring the event.`,
+ {
+ repository: `${repositoryOwner}/${repositoryName}`,
+ messageId,
+ },
+ );
+
+ continue;
+ }
+
+ const key = enableOrgLevel ? payload.repositoryOwner : `${payload.repositoryOwner}/${payload.repositoryName}`;
+
+ let entry = validMessages.get(key);
+
+ // If we've not seen this owner/repo before, we'll need to create a GitHub
+ // client for it.
+ if (entry === undefined) {
+ const installationId = await getInstallationId(githubAppClient, enableOrgLevel, payload);
+ const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl);
+ const githubInstallationClient = await createOctokitClient(ghAuth.token, ghesApiUrl);
+
+ entry = {
+ messages: [],
+ githubInstallationClient,
+ };
+
+ validMessages.set(key, entry);
+ }
+
+ entry.messages.push(payload);
}
- const ephemeral = ephemeralEnabled && payload.eventType === 'workflow_job';
const runnerType = enableOrgLevel ? 'Org' : 'Repo';
- const runnerOwner = enableOrgLevel ? payload.repositoryOwner : `${payload.repositoryOwner}/${payload.repositoryName}`;
addPersistentContextToChildLogger({
runner: {
+ ephemeral: ephemeralEnabled,
type: runnerType,
- owner: runnerOwner,
namePrefix: runnerNamePrefix,
- },
- github: {
- event: payload.eventType,
- workflow_job_id: payload.id.toString(),
+ n_events: Array.from(validMessages.values()).reduce((acc, group) => acc + group.messages.length, 0),
},
});
- logger.info(`Received event`);
+ logger.info(`Received events`);
- const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl();
+ for (const [group, { githubInstallationClient, messages }] of validMessages.entries()) {
+ // Work out how much we want to scale up by.
+ let scaleUp = 0;
- const installationId = await getInstallationId(ghesApiUrl, enableOrgLevel, payload);
- const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl);
- const githubInstallationClient = await createOctokitClient(ghAuth.token, ghesApiUrl);
+ for (const message of messages) {
+ const messageLogger = logger.createChild({
+ persistentKeys: {
+ eventType: message.eventType,
+ group,
+ messageId: message.messageId,
+ repository: `${message.repositoryOwner}/${message.repositoryName}`,
+ },
+ });
- if (!enableJobQueuedCheck || (await isJobQueued(githubInstallationClient, payload))) {
- let scaleUp = true;
- if (maximumRunners !== -1) {
- const currentRunners = await listEC2Runners({
- environment,
- runnerType,
- runnerOwner,
+ if (enableJobQueuedCheck && !(await isJobQueued(githubInstallationClient, message))) {
+ messageLogger.info('No runner will be created, job is not queued.');
+
+ continue;
+ }
+
+ scaleUp++;
+ }
+
+ if (scaleUp === 0) {
+ logger.info('No runners will be created for this group, no valid messages found.');
+
+ continue;
+ }
+
+ // Don't call the EC2 API if we can create an unlimited number of runners.
+ const currentRunners =
+ maximumRunners === -1 ? 0 : (await listEC2Runners({ environment, runnerType, runnerOwner: group })).length;
+
+ logger.info('Current runners', {
+ currentRunners,
+ maximumRunners,
+ });
+
+ // Calculate how many runners we want to create.
+ const newRunners =
+ maximumRunners === -1
+ ? // If we don't have an upper limit, scale up by the number of new jobs.
+ scaleUp
+ : // Otherwise, we do have a limit, so work out if `scaleUp` would exceed it.
+ Math.min(scaleUp, maximumRunners - currentRunners);
+
+ const missingInstanceCount = Math.max(0, scaleUp - newRunners);
+
+ if (missingInstanceCount > 0) {
+ logger.info('Not all runners will be created for this group, maximum number of runners reached.', {
+ desiredNewRunners: scaleUp,
});
- logger.info(`Current runners: ${currentRunners.length} of ${maximumRunners}`);
- scaleUp = currentRunners.length < maximumRunners;
+
+ if (ephemeralEnabled) {
+ // This removes `missingInstanceCount` items from the start of the array
+ // so that, if we retry more messages later, we pick fresh ones.
+ invalidMessages.push(...messages.splice(0, missingInstanceCount).map(({ messageId }) => messageId));
+ }
+
+ // No runners will be created, so skip calling the EC2 API.
+ if (missingInstanceCount === scaleUp) {
+ continue;
+ }
}
- if (scaleUp) {
- logger.info(`Attempting to launch a new runner`);
+ logger.info(`Attempting to launch new runners`, {
+ newRunners,
+ });
- await createRunners(
- {
- ephemeral,
- enableJitConfig,
- ghesBaseUrl,
- runnerLabels,
- runnerGroup,
- runnerNamePrefix,
- runnerOwner,
- runnerType,
- disableAutoUpdate,
- ssmTokenPath,
- ssmConfigPath,
- },
- {
- ec2instanceCriteria: {
- instanceTypes,
- targetCapacityType: instanceTargetCapacityType,
- maxSpotPrice: instanceMaxSpotPrice,
- instanceAllocationStrategy: instanceAllocationStrategy,
- },
- environment,
- launchTemplateName,
- subnets,
- amiIdSsmParameterName,
- tracingEnabled,
- onDemandFailoverOnError,
+ const instances = await createRunners(
+ {
+ ephemeral: ephemeralEnabled,
+ enableJitConfig,
+ ghesBaseUrl,
+ runnerLabels,
+ runnerGroup,
+ runnerNamePrefix,
+ runnerOwner: group,
+ runnerType,
+ disableAutoUpdate,
+ ssmTokenPath,
+ ssmConfigPath,
+ },
+ {
+ ec2instanceCriteria: {
+ instanceTypes,
+ targetCapacityType: instanceTargetCapacityType,
+ maxSpotPrice: instanceMaxSpotPrice,
+ instanceAllocationStrategy: instanceAllocationStrategy,
},
- githubInstallationClient,
- );
+ environment,
+ launchTemplateName,
+ subnets,
+ amiIdSsmParameterName,
+ tracingEnabled,
+ onDemandFailoverOnError,
+ },
+ newRunners,
+ githubInstallationClient,
+ );
- await publishRetryMessage(payload);
- } else {
- logger.info('No runner will be created, maximum number of runners reached.');
- if (ephemeral) {
- throw new ScaleError('No runners create: maximum of runners reached.');
- }
+ // Not all runners we wanted were created, let's reject enough items so that
+ // number of entries will be retried.
+ if (instances.length !== newRunners) {
+ const failedInstanceCount = newRunners - instances.length;
+
+ logger.warn('Some runners failed to be created, rejecting some messages so the requests are retried', {
+ wanted: newRunners,
+ got: instances.length,
+ failedInstanceCount,
+ });
+
+ invalidMessages.push(...messages.slice(0, failedInstanceCount).map(({ messageId }) => messageId));
}
- } else {
- logger.info('No runner will be created, job is not queued.');
}
+
+ return invalidMessages;
}
export function getGitHubEnterpriseApiUrl() {
diff --git a/lambdas/libs/aws-powertools-util/src/logger/index.ts b/lambdas/libs/aws-powertools-util/src/logger/index.ts
index 195b552a74..2bad191a83 100644
--- a/lambdas/libs/aws-powertools-util/src/logger/index.ts
+++ b/lambdas/libs/aws-powertools-util/src/logger/index.ts
@@ -9,7 +9,7 @@ const defaultValues = {
};
function setContext(context: Context, module?: string) {
- logger.addPersistentLogAttributes({
+ logger.appendPersistentKeys({
'aws-request-id': context.awsRequestId,
'function-name': context.functionName,
module: module,
@@ -17,7 +17,7 @@ function setContext(context: Context, module?: string) {
// Add the context to all child loggers
childLoggers.forEach((childLogger) => {
- childLogger.addPersistentLogAttributes({
+ childLogger.appendPersistentKeys({
'aws-request-id': context.awsRequestId,
'function-name': context.functionName,
});
@@ -25,14 +25,14 @@ function setContext(context: Context, module?: string) {
}
const logger = new Logger({
- persistentLogAttributes: {
+ persistentKeys: {
...defaultValues,
},
});
function createChildLogger(module: string): Logger {
const childLogger = logger.createChild({
- persistentLogAttributes: {
+ persistentKeys: {
module: module,
},
});
@@ -47,7 +47,7 @@ type LogAttributes = {
function addPersistentContextToChildLogger(attributes: LogAttributes) {
childLoggers.forEach((childLogger) => {
- childLogger.addPersistentLogAttributes(attributes);
+ childLogger.appendPersistentKeys(attributes);
});
}
diff --git a/main.tf b/main.tf
index 74c4c54ec6..f0dadd6b66 100644
--- a/main.tf
+++ b/main.tf
@@ -210,28 +210,30 @@ module "runners" {
credit_specification = var.runner_credit_specification
cpu_options = var.runner_cpu_options
- enable_runner_binaries_syncer = var.enable_runner_binaries_syncer
- lambda_s3_bucket = var.lambda_s3_bucket
- runners_lambda_s3_key = var.runners_lambda_s3_key
- runners_lambda_s3_object_version = var.runners_lambda_s3_object_version
- lambda_runtime = var.lambda_runtime
- lambda_architecture = var.lambda_architecture
- lambda_zip = var.runners_lambda_zip
- lambda_scale_up_memory_size = var.runners_scale_up_lambda_memory_size
- lambda_scale_down_memory_size = var.runners_scale_down_lambda_memory_size
- lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout
- lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout
- lambda_subnet_ids = var.lambda_subnet_ids
- lambda_security_group_ids = var.lambda_security_group_ids
- lambda_tags = var.lambda_tags
- tracing_config = var.tracing_config
- logging_retention_in_days = var.logging_retention_in_days
- logging_kms_key_id = var.logging_kms_key_id
- enable_cloudwatch_agent = var.enable_cloudwatch_agent
- cloudwatch_config = var.cloudwatch_config
- runner_log_files = var.runner_log_files
- runner_group_name = var.runner_group_name
- runner_name_prefix = var.runner_name_prefix
+ enable_runner_binaries_syncer = var.enable_runner_binaries_syncer
+ lambda_s3_bucket = var.lambda_s3_bucket
+ runners_lambda_s3_key = var.runners_lambda_s3_key
+ runners_lambda_s3_object_version = var.runners_lambda_s3_object_version
+ lambda_runtime = var.lambda_runtime
+ lambda_architecture = var.lambda_architecture
+ lambda_event_source_mapping_batch_size = var.lambda_event_source_mapping_batch_size
+ lambda_event_source_mapping_maximum_batching_window_in_seconds = var.lambda_event_source_mapping_maximum_batching_window_in_seconds
+ lambda_zip = var.runners_lambda_zip
+ lambda_scale_up_memory_size = var.runners_scale_up_lambda_memory_size
+ lambda_scale_down_memory_size = var.runners_scale_down_lambda_memory_size
+ lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout
+ lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout
+ lambda_subnet_ids = var.lambda_subnet_ids
+ lambda_security_group_ids = var.lambda_security_group_ids
+ lambda_tags = var.lambda_tags
+ tracing_config = var.tracing_config
+ logging_retention_in_days = var.logging_retention_in_days
+ logging_kms_key_id = var.logging_kms_key_id
+ enable_cloudwatch_agent = var.enable_cloudwatch_agent
+ cloudwatch_config = var.cloudwatch_config
+ runner_log_files = var.runner_log_files
+ runner_group_name = var.runner_group_name
+ runner_name_prefix = var.runner_name_prefix
scale_up_reserved_concurrent_executions = var.scale_up_reserved_concurrent_executions
diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md
index 198078feee..0515763f4c 100644
--- a/modules/multi-runner/README.md
+++ b/modules/multi-runner/README.md
@@ -137,6 +137,8 @@ module "multi-runner" {
| [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no |
| [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. | `string` | `null` | no |
| [lambda\_architecture](#input\_lambda\_architecture) | AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions. | `string` | `"arm64"` | no |
+| [lambda\_event\_source\_mapping\_batch\_size](#input\_lambda\_event\_source\_mapping\_batch\_size) | Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default of 10 events will be used. | `number` | `10` | no |
+| [lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds](#input\_lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds) | Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch\_size is greater than 10. Defaults to 0. | `number` | `0` | no |
| [lambda\_principals](#input\_lambda\_principals) | (Optional) add extra principals to the role created for execution of the lambda, e.g. for local testing. | list(object({
type = string
identifiers = list(string)
})) | `[]` | no |
| [lambda\_runtime](#input\_lambda\_runtime) | AWS Lambda runtime. | `string` | `"nodejs22.x"` | no |
| [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `string` | `null` | no |
diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf
index 811ab36260..d58e61f6ac 100644
--- a/modules/multi-runner/runners.tf
+++ b/modules/multi-runner/runners.tf
@@ -58,28 +58,30 @@ module "runners" {
credit_specification = each.value.runner_config.credit_specification
cpu_options = each.value.runner_config.cpu_options
- enable_runner_binaries_syncer = each.value.runner_config.enable_runner_binaries_syncer
- lambda_s3_bucket = var.lambda_s3_bucket
- runners_lambda_s3_key = var.runners_lambda_s3_key
- runners_lambda_s3_object_version = var.runners_lambda_s3_object_version
- lambda_runtime = var.lambda_runtime
- lambda_architecture = var.lambda_architecture
- lambda_zip = var.runners_lambda_zip
- lambda_scale_up_memory_size = var.scale_up_lambda_memory_size
- lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout
- lambda_scale_down_memory_size = var.scale_down_lambda_memory_size
- lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout
- lambda_subnet_ids = var.lambda_subnet_ids
- lambda_security_group_ids = var.lambda_security_group_ids
- lambda_tags = var.lambda_tags
- tracing_config = var.tracing_config
- logging_retention_in_days = var.logging_retention_in_days
- logging_kms_key_id = var.logging_kms_key_id
- enable_cloudwatch_agent = each.value.runner_config.enable_cloudwatch_agent
- cloudwatch_config = try(coalesce(each.value.runner_config.cloudwatch_config, var.cloudwatch_config), null)
- runner_log_files = each.value.runner_config.runner_log_files
- runner_group_name = each.value.runner_config.runner_group_name
- runner_name_prefix = each.value.runner_config.runner_name_prefix
+ enable_runner_binaries_syncer = each.value.runner_config.enable_runner_binaries_syncer
+ lambda_s3_bucket = var.lambda_s3_bucket
+ runners_lambda_s3_key = var.runners_lambda_s3_key
+ runners_lambda_s3_object_version = var.runners_lambda_s3_object_version
+ lambda_runtime = var.lambda_runtime
+ lambda_architecture = var.lambda_architecture
+ lambda_zip = var.runners_lambda_zip
+ lambda_scale_up_memory_size = var.scale_up_lambda_memory_size
+ lambda_event_source_mapping_batch_size = var.lambda_event_source_mapping_batch_size
+ lambda_event_source_mapping_maximum_batching_window_in_seconds = var.lambda_event_source_mapping_maximum_batching_window_in_seconds
+ lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout
+ lambda_scale_down_memory_size = var.scale_down_lambda_memory_size
+ lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout
+ lambda_subnet_ids = var.lambda_subnet_ids
+ lambda_security_group_ids = var.lambda_security_group_ids
+ lambda_tags = var.lambda_tags
+ tracing_config = var.tracing_config
+ logging_retention_in_days = var.logging_retention_in_days
+ logging_kms_key_id = var.logging_kms_key_id
+ enable_cloudwatch_agent = each.value.runner_config.enable_cloudwatch_agent
+ cloudwatch_config = try(coalesce(each.value.runner_config.cloudwatch_config, var.cloudwatch_config), null)
+ runner_log_files = each.value.runner_config.runner_log_files
+ runner_group_name = each.value.runner_config.runner_group_name
+ runner_name_prefix = each.value.runner_config.runner_name_prefix
scale_up_reserved_concurrent_executions = each.value.runner_config.scale_up_reserved_concurrent_executions
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
index 119af5c36c..5c839e1104 100644
--- a/modules/multi-runner/variables.tf
+++ b/modules/multi-runner/variables.tf
@@ -724,3 +724,15 @@ variable "user_agent" {
type = string
default = "github-aws-runners"
}
+
+variable "lambda_event_source_mapping_batch_size" {
+ description = "Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default of 10 events will be used."
+ type = number
+ default = 10
+}
+
+variable "lambda_event_source_mapping_maximum_batching_window_in_seconds" {
+ description = "Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch_size is greater than 10. Defaults to 0."
+ type = number
+ default = 0
+}
diff --git a/modules/runners/README.md b/modules/runners/README.md
index eddf7edb40..169cee7eac 100644
--- a/modules/runners/README.md
+++ b/modules/runners/README.md
@@ -177,6 +177,8 @@ yarn run dist
| [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no |
| [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. | `string` | `null` | no |
| [lambda\_architecture](#input\_lambda\_architecture) | AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions. | `string` | `"arm64"` | no |
+| [lambda\_event\_source\_mapping\_batch\_size](#input\_lambda\_event\_source\_mapping\_batch\_size) | Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default of 10 events will be used. | `number` | `10` | no |
+| [lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds](#input\_lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds) | Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch\_size is greater than 10. Defaults to 0. | `number` | `0` | no |
| [lambda\_runtime](#input\_lambda\_runtime) | AWS Lambda runtime. | `string` | `"nodejs22.x"` | no |
| [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `string` | `null` | no |
| [lambda\_scale\_down\_memory\_size](#input\_lambda\_scale\_down\_memory\_size) | Memory size limit in MB for scale down lambda. | `number` | `512` | no |
diff --git a/modules/runners/job-retry.tf b/modules/runners/job-retry.tf
index e51c3903d4..130992667f 100644
--- a/modules/runners/job-retry.tf
+++ b/modules/runners/job-retry.tf
@@ -3,30 +3,32 @@ locals {
job_retry_enabled = var.job_retry != null && var.job_retry.enable ? true : false
job_retry = {
- prefix = var.prefix
- tags = local.tags
- aws_partition = var.aws_partition
- architecture = var.lambda_architecture
- runtime = var.lambda_runtime
- security_group_ids = var.lambda_security_group_ids
- subnet_ids = var.lambda_subnet_ids
- kms_key_arn = var.kms_key_arn
- lambda_tags = var.lambda_tags
- log_level = var.log_level
- logging_kms_key_id = var.logging_kms_key_id
- logging_retention_in_days = var.logging_retention_in_days
- metrics = var.metrics
- role_path = var.role_path
- role_permissions_boundary = var.role_permissions_boundary
- s3_bucket = var.lambda_s3_bucket
- s3_key = var.runners_lambda_s3_key
- s3_object_version = var.runners_lambda_s3_object_version
- zip = var.lambda_zip
- tracing_config = var.tracing_config
- github_app_parameters = var.github_app_parameters
- enable_organization_runners = var.enable_organization_runners
- sqs_build_queue = var.sqs_build_queue
- ghes_url = var.ghes_url
+ prefix = var.prefix
+ tags = local.tags
+ aws_partition = var.aws_partition
+ architecture = var.lambda_architecture
+ runtime = var.lambda_runtime
+ security_group_ids = var.lambda_security_group_ids
+ subnet_ids = var.lambda_subnet_ids
+ kms_key_arn = var.kms_key_arn
+ lambda_tags = var.lambda_tags
+ log_level = var.log_level
+ logging_kms_key_id = var.logging_kms_key_id
+ logging_retention_in_days = var.logging_retention_in_days
+ metrics = var.metrics
+ role_path = var.role_path
+ role_permissions_boundary = var.role_permissions_boundary
+ s3_bucket = var.lambda_s3_bucket
+ s3_key = var.runners_lambda_s3_key
+ s3_object_version = var.runners_lambda_s3_object_version
+ zip = var.lambda_zip
+ tracing_config = var.tracing_config
+ github_app_parameters = var.github_app_parameters
+ enable_organization_runners = var.enable_organization_runners
+ sqs_build_queue = var.sqs_build_queue
+ ghes_url = var.ghes_url
+ lambda_event_source_mapping_batch_size = var.lambda_event_source_mapping_batch_size
+ lambda_event_source_mapping_maximum_batching_window_in_seconds = var.lambda_event_source_mapping_maximum_batching_window_in_seconds
}
}
diff --git a/modules/runners/job-retry/README.md b/modules/runners/job-retry/README.md
index 168f2d324e..f54b943855 100644
--- a/modules/runners/job-retry/README.md
+++ b/modules/runners/job-retry/README.md
@@ -42,7 +42,7 @@ The module is an inner module and used by the runner module when the opt-in feat
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
-| [config](#input\_config) | Configuration for the spot termination watcher lambda function.
`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
`enable_organization_runners`: Enable organization runners.
`enable_metric`: Enable metric for the lambda. If `spot_warning` is set to true, the lambda will emit a metric when it detects a spot termination warning.
'ghes\_url': Optional GitHub Enterprise Server URL.
'user\_agent': Optional User-Agent header for GitHub API requests.
'github\_app\_parameters': Parameter Store for GitHub App Parameters.
'kms\_key\_arn': Optional CMK Key ARN instead of using the default AWS managed key.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`metrics`: Configuration to enable metrics creation by the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
'sqs\_build\_queue': SQS queue for build events to re-publish job request.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file. | object({
aws_partition = optional(string, null)
architecture = optional(string, null)
enable_organization_runners = bool
environment_variables = optional(map(string), {})
ghes_url = optional(string, null)
user_agent = optional(string, null)
github_app_parameters = object({
key_base64 = map(string)
id = map(string)
})
kms_key_arn = optional(string, null)
lambda_tags = optional(map(string), {})
log_level = optional(string, null)
logging_kms_key_id = optional(string, null)
logging_retention_in_days = optional(number, null)
memory_size = optional(number, null)
metrics = optional(object({
enable = optional(bool, false)
namespace = optional(string, null)
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
}), {})
}), {})
prefix = optional(string, null)
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
queue_encryption = optional(object({
kms_data_key_reuse_period_seconds = optional(number, null)
kms_master_key_id = optional(string, null)
sqs_managed_sse_enabled = optional(bool, true)
}), {})
role_path = optional(string, null)
role_permissions_boundary = optional(string, null)
runtime = optional(string, null)
security_group_ids = optional(list(string), [])
subnet_ids = optional(list(string), [])
s3_bucket = optional(string, null)
s3_key = optional(string, null)
s3_object_version = optional(string, null)
sqs_build_queue = object({
url = string
arn = string
})
tags = optional(map(string), {})
timeout = optional(number, 30)
tracing_config = optional(object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
}), {})
zip = optional(string, null)
}) | n/a | yes |
+| [config](#input\_config) | Configuration for the spot termination watcher lambda function.
`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
`enable_organization_runners`: Enable organization runners.
`enable_metric`: Enable metric for the lambda. If `spot_warning` is set to true, the lambda will emit a metric when it detects a spot termination warning.
'ghes\_url': Optional GitHub Enterprise Server URL.
'user\_agent': Optional User-Agent header for GitHub API requests.
'github\_app\_parameters': Parameter Store for GitHub App Parameters.
'kms\_key\_arn': Optional CMK Key ARN instead of using the default AWS managed key.
`lambda_event_source_mapping_batch_size`: Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default will be used.
`lambda_event_source_mapping_maximum_batching_window_in_seconds`: Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch\_size is greater than 10.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`metrics`: Configuration to enable metrics creation by the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
'sqs\_build\_queue': SQS queue for build events to re-publish job request.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file. | object({
aws_partition = optional(string, null)
architecture = optional(string, null)
enable_organization_runners = bool
environment_variables = optional(map(string), {})
ghes_url = optional(string, null)
user_agent = optional(string, null)
github_app_parameters = object({
key_base64 = map(string)
id = map(string)
})
kms_key_arn = optional(string, null)
lambda_event_source_mapping_batch_size = optional(number, 10)
lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, 0)
lambda_tags = optional(map(string), {})
log_level = optional(string, null)
logging_kms_key_id = optional(string, null)
logging_retention_in_days = optional(number, null)
memory_size = optional(number, null)
metrics = optional(object({
enable = optional(bool, false)
namespace = optional(string, null)
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
}), {})
}), {})
prefix = optional(string, null)
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
queue_encryption = optional(object({
kms_data_key_reuse_period_seconds = optional(number, null)
kms_master_key_id = optional(string, null)
sqs_managed_sse_enabled = optional(bool, true)
}), {})
role_path = optional(string, null)
role_permissions_boundary = optional(string, null)
runtime = optional(string, null)
security_group_ids = optional(list(string), [])
subnet_ids = optional(list(string), [])
s3_bucket = optional(string, null)
s3_key = optional(string, null)
s3_object_version = optional(string, null)
sqs_build_queue = object({
url = string
arn = string
})
tags = optional(map(string), {})
timeout = optional(number, 30)
tracing_config = optional(object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
}), {})
zip = optional(string, null)
}) | n/a | yes |
## Outputs
diff --git a/modules/runners/job-retry/main.tf b/modules/runners/job-retry/main.tf
index 807f52a49a..eba478b214 100644
--- a/modules/runners/job-retry/main.tf
+++ b/modules/runners/job-retry/main.tf
@@ -44,9 +44,10 @@ module "job_retry" {
}
resource "aws_lambda_event_source_mapping" "job_retry" {
- event_source_arn = aws_sqs_queue.job_retry_check_queue.arn
- function_name = module.job_retry.lambda.function.arn
- batch_size = 1
+ event_source_arn = aws_sqs_queue.job_retry_check_queue.arn
+ function_name = module.job_retry.lambda.function.arn
+ batch_size = var.config.lambda_event_source_mapping_batch_size
+ maximum_batching_window_in_seconds = var.config.lambda_event_source_mapping_maximum_batching_window_in_seconds
}
resource "aws_lambda_permission" "job_retry" {
diff --git a/modules/runners/job-retry/variables.tf b/modules/runners/job-retry/variables.tf
index 4741dd1b45..7ccfdf63b3 100644
--- a/modules/runners/job-retry/variables.tf
+++ b/modules/runners/job-retry/variables.tf
@@ -11,6 +11,8 @@ variable "config" {
'user_agent': Optional User-Agent header for GitHub API requests.
'github_app_parameters': Parameter Store for GitHub App Parameters.
'kms_key_arn': Optional CMK Key ARN instead of using the default AWS managed key.
+ `lambda_event_source_mapping_batch_size`: Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default will be used.
+ `lambda_event_source_mapping_maximum_batching_window_in_seconds`: Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch_size is greater than 10.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
@@ -45,12 +47,14 @@ variable "config" {
key_base64 = map(string)
id = map(string)
})
- kms_key_arn = optional(string, null)
- lambda_tags = optional(map(string), {})
- log_level = optional(string, null)
- logging_kms_key_id = optional(string, null)
- logging_retention_in_days = optional(number, null)
- memory_size = optional(number, null)
+ kms_key_arn = optional(string, null)
+ lambda_event_source_mapping_batch_size = optional(number, 10)
+ lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, 0)
+ lambda_tags = optional(map(string), {})
+ log_level = optional(string, null)
+ logging_kms_key_id = optional(string, null)
+ logging_retention_in_days = optional(number, null)
+ memory_size = optional(number, null)
metrics = optional(object({
enable = optional(bool, false)
namespace = optional(string, null)
diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf
index 89d95a50d0..b1ea88652d 100644
--- a/modules/runners/scale-up.tf
+++ b/modules/runners/scale-up.tf
@@ -87,10 +87,12 @@ resource "aws_cloudwatch_log_group" "scale_up" {
}
resource "aws_lambda_event_source_mapping" "scale_up" {
- event_source_arn = var.sqs_build_queue.arn
- function_name = aws_lambda_function.scale_up.arn
- batch_size = 1
- tags = var.tags
+ event_source_arn = var.sqs_build_queue.arn
+ function_name = aws_lambda_function.scale_up.arn
+ function_response_types = ["ReportBatchItemFailures"]
+ batch_size = var.lambda_event_source_mapping_batch_size
+ maximum_batching_window_in_seconds = var.lambda_event_source_mapping_maximum_batching_window_in_seconds
+ tags = var.tags
}
resource "aws_lambda_permission" "scale_runners_lambda" {
diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf
index 352285e786..a45075bb52 100644
--- a/modules/runners/variables.tf
+++ b/modules/runners/variables.tf
@@ -770,3 +770,23 @@ variable "user_agent" {
type = string
default = null
}
+
+variable "lambda_event_source_mapping_batch_size" {
+ description = "Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default of 10 events will be used."
+ type = number
+ default = 10
+ validation {
+ condition = var.lambda_event_source_mapping_batch_size >= 1 && var.lambda_event_source_mapping_batch_size <= 1000
+ error_message = "The batch size for the lambda event source mapping must be between 1 and 1000."
+ }
+}
+
+variable "lambda_event_source_mapping_maximum_batching_window_in_seconds" {
+ description = "Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch_size is greater than 10. Defaults to 0."
+ type = number
+ default = 0
+ validation {
+ condition = var.lambda_event_source_mapping_maximum_batching_window_in_seconds >= 0 && var.lambda_event_source_mapping_maximum_batching_window_in_seconds <= 300
+ error_message = "Maximum batching window must be between 0 and 300 seconds."
+ }
+}
diff --git a/modules/webhook-github-app/README.md b/modules/webhook-github-app/README.md
index 0c09a761c5..6de85ee30d 100644
--- a/modules/webhook-github-app/README.md
+++ b/modules/webhook-github-app/README.md
@@ -34,7 +34,7 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
-| [github\_app](#input\_github\_app) | GitHub app parameters, see your github app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`). | object({
key_base64 = string
id = string
webhook_secret = string
}) | n/a | yes |
+| [github\_app](#input\_github\_app) | GitHub app parameters, see your GitHub app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`). | object({
key_base64 = string
id = string
webhook_secret = string
}) | n/a | yes |
| [webhook\_endpoint](#input\_webhook\_endpoint) | The endpoint to use for the webhook, defaults to the endpoint of the runners module. | `string` | n/a | yes |
## Outputs
diff --git a/variables.tf b/variables.tf
index 0bf3563145..7ff6ecece4 100644
--- a/variables.tf
+++ b/variables.tf
@@ -1021,3 +1021,19 @@ variable "user_agent" {
type = string
default = "github-aws-runners"
}
+
+variable "lambda_event_source_mapping_batch_size" {
+ description = "Maximum number of records to pass to the lambda function in a single batch for the event source mapping. When not set, the AWS default of 10 events will be used."
+ type = number
+ default = 10
+}
+
+variable "lambda_event_source_mapping_maximum_batching_window_in_seconds" {
+ description = "Maximum amount of time to gather records before invoking the lambda function, in seconds. AWS requires this to be greater than 0 if batch_size is greater than 10. Defaults to 0."
+ type = number
+ default = 0
+ validation {
+ condition = var.lambda_event_source_mapping_maximum_batching_window_in_seconds >= 0 && var.lambda_event_source_mapping_maximum_batching_window_in_seconds <= 300
+ error_message = "Maximum batching window must be between 0 and 300 seconds."
+ }
+}