diff --git a/README.md b/README.md index 7cbadd9..0f9f0c0 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ This module supports all 4 OSes, but support for detailed configuration of the n are fully supported for Amazon Linux 2 and Windows, and take advantage of the [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh) supplied on those AMIs. **NONE** of these inputs are supported on Bottlerocket. On AL2023, only the first 2 are supported. +Note: The fourth option is only available when setting an AMI ID for the node group; otherwise AWS will default to the released AMI for the OS and EKS will append your userdata with the bootstrap script. Note that for all OSes, you can supply the complete `userdata` contents, which will be untouched by this module, via `userdata_override_base64`. @@ -318,14 +319,14 @@ module "eks_node_group" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | -| [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | +| [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional (bash for linux, powershell for Windows) commands to execute on each worker node after joining the EKS cluster
(after executing the `bootstrap` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production"
This can only be used with AL2 and Windows AMI types with a custom AMI set as EKS will inject a bootstrapper into all other user-scripts | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use, overriding other AMI specifications, but must match `ami_type`. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | The EKS AMI "release version" to use. Defaults to the latest recommended version.
For Amazon Linux, it is the "Release version" from [Amazon AMI Releases](https://github.com/awslabs/amazon-eks-ami/releases)
For Bottlerocket, it is the release tag from [Bottlerocket Releases](https://github.com/bottlerocket-os/bottlerocket/releases) without the "v" prefix.
For Windows, it is "AMI version" from [AWS docs](https://docs.aws.amazon.com/eks/latest/userguide/eks-ami-versions-windows.html).
Note that unlike AMI names, release versions never include the "v" prefix.
Examples:
AL2: 1.29.3-20240531
Bottlerocket: 1.2.0 or 1.2.0-ccf1b754
Windows: 1.29-2024.04.09 | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_FIPS, BOTTLEROCKET_x86_64_FIPS, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64, AL2023_x86_64_STANDARD, AL2023_ARM_64_STANDARD, AL2023_x86_64_NEURON, AL2023_x86_64_NVIDIA`. | `string` | `"AL2_x86_64"` | no | | [associate\_cluster\_security\_group](#input\_associate\_cluster\_security\_group) | When true, associate the default cluster security group to the nodes. If disabled the EKS managed security group will not
be associated to the nodes and you will need to provide another security group that allows the nodes to communicate with
the EKS control plane. Be aware that if no `associated_security_group_ids` or `ssh_access_security_group_ids` are provided,
then the nodes will have no inbound or outbound rules. | `bool` | `true` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | -| [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | +| [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh/ps1` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [block\_device\_map](#input\_block\_device\_map) | Map of block device name specification, see [launch\_template.block-devices](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template#block-devices). | 
map(object({
    no_device    = optional(bool, null)
    virtual_name = optional(string, null)
    ebs = optional(object({
      delete_on_termination = optional(bool, true)
      encrypted             = optional(bool, true)
      iops                  = optional(number, null)
      kms_key_id            = optional(string, null)
      snapshot_id           = optional(string, null)
      throughput            = optional(number, null)
      volume_size           = optional(number, 20)
      volume_type           = optional(string, "gp3")
    }))
  }))
| 
{
  "/dev/xvda": {
    "ebs": {}
  }
}
| no | | [block\_device\_mappings](#input\_block\_device\_mappings) | DEPRECATED: Use `block_device_map` instead.
List of block device mappings for the launch template.
Each list element is an object with a `device_name` key and
any keys supported by the `ebs` block of `launch_template`. | `list(any)` | `null` | no | | [bootstrap\_additional\_options](#input\_bootstrap\_additional\_options) | Additional options to bootstrap.sh. DO NOT include `--kubelet-additional-args`, use `kubelet_additional_options` var instead. Not used with AL2023 AMI types. | `list(string)` | `[]` | no | diff --git a/README.yaml b/README.yaml index c8c0613..e465e71 100644 --- a/README.yaml +++ b/README.yaml @@ -101,6 +101,7 @@ introduction: |- are fully supported for Amazon Linux 2 and Windows, and take advantage of the [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh) supplied on those AMIs. **NONE** of these inputs are supported on Bottlerocket. On AL2023, only the first 2 are supported. + Note: The fourth option is only available when setting an AMI ID for the node group; otherwise AWS will default to the released AMI for the OS and EKS will append your userdata with the bootstrap script. Note that for all OSes, you can supply the complete `userdata` contents, which will be untouched by this module, via `userdata_override_base64`. diff --git a/launch-template.tf b/launch-template.tf index 3af389b..35d5bfe 100644 --- a/launch-template.tf +++ b/launch-template.tf @@ -197,6 +197,10 @@ resource "aws_launch_template" "default" { condition = contains(["AL2", "WINDOWS"], local.ami_os) || length(local.userdata_vars.after_cluster_joining_userdata) == 0 || (local.ami_os == "AL2" || local.ami_os == "WINDOWS") error_message = format("The input `after_cluster_joining_userdata` is not supported for %v.", title(lower(local.ami_os))) } + precondition { + condition = length(local.userdata_vars.after_cluster_joining_userdata) == 0 || length(var.ami_image_id) != 0 && length(local.userdata_vars.after_cluster_joining_userdata) > 0 && (local.ami_os == "AL2" || local.ami_os == "WINDOWS") + error_message = format("The input `after_cluster_joining_userdata` is not supported for %v, a custom ami_image_id must be set for this functionality", title(lower(local.ami_os))) + } } } diff --git a/userdata.tf b/userdata.tf index 31d4aed..06541a8 100644 --- a/userdata.tf +++ b/userdata.tf @@ -49,7 +49,11 @@ locals { WINDOWS = "${path.module}/userdata_nt.tpl" } - + userdata_bootstrapper_template_file = { + AL2 = "${path.module}/userdata_bootstrap.tpl" + BOTTLEROCKET = "${path.module}/userdata_bootstrap.tpl" + WINDOWS = "${path.module}/userdata_bootstrap_nt.tpl" + } # When suppressing EKS bootstrap, add --register-with-taints to kubelet_extra_args, # e.g. --register-with-taints=test=:PreferNoSchedule @@ -71,6 +75,7 @@ locals { kubelet_extra_args_yaml = replace(local.kubelet_extra_args, "--", "\n - >-\n --") userdata_vars = { + bootstrap_script = (length(var.ami_image_id) > 0 && length(var.after_cluster_joining_userdata) > 0 && (local.ami_os == "AL2" || local.ami_os == "WINDOWS")) ? file(local.userdata_bootstrapper_template_file[local.ami_os]) : "" before_cluster_joining_userdata = length(var.before_cluster_joining_userdata) == 0 ? "" : join("\n", var.before_cluster_joining_userdata) kubelet_extra_args = local.kubelet_extra_args kubelet_extra_args_yaml = local.kubelet_extra_args_yaml diff --git a/userdata.tpl b/userdata.tpl index 426eec4..ec97948 100644 --- a/userdata.tpl +++ b/userdata.tpl @@ -15,7 +15,7 @@ export KUBELET_EXTRA_ARGS="${kubelet_extra_args}" %{ endif } %{ if length(kubelet_extra_args) > 0 || length (bootstrap_extra_args) > 0 || length (after_cluster_joining_userdata) > 0 } -/etc/eks/bootstrap.sh --apiserver-endpoint '${cluster_endpoint}' --b64-cluster-ca '${certificate_authority_data}' ${bootstrap_extra_args} '${cluster_name}' +${bootstrap_script} ${after_cluster_joining_userdata} %{ endif } diff --git a/userdata_bootstrap.tpl b/userdata_bootstrap.tpl new file mode 100644 index 0000000..287df57 --- /dev/null +++ b/userdata_bootstrap.tpl @@ -0,0 +1 @@ +/etc/eks/bootstrap.sh --apiserver-endpoint '${cluster_endpoint}' --b64-cluster-ca '${certificate_authority_data}' ${bootstrap_extra_args} '${cluster_name}' diff --git a/userdata_bootstrap_nt.tpl b/userdata_bootstrap_nt.tpl new file mode 100644 index 0000000..a2844c7 --- /dev/null +++ b/userdata_bootstrap_nt.tpl @@ -0,0 +1,5 @@ +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" + +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" ${bootstrap_extra_args} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 diff --git a/userdata_nt.tpl b/userdata_nt.tpl index 5507816..7bae7e5 100644 --- a/userdata_nt.tpl +++ b/userdata_nt.tpl @@ -26,7 +26,7 @@ if ($disks_to_adjust -ne $null) { [string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" -& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" ${bootstrap_extra_args} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 +${bootstrap_script} try{ ${after_cluster_joining_userdata} @@ -34,4 +34,5 @@ ${after_cluster_joining_userdata} Write-Host "An error occurred in post-script" -ForegroundColor Red Write-Host $_.ScriptStackTrace } + diff --git a/variables.tf b/variables.tf index a4b798e..00f74a6 100644 --- a/variables.tf +++ b/variables.tf @@ -358,14 +358,18 @@ variable "resources_to_tag" { variable "before_cluster_joining_userdata" { type = list(string) - description = "Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" + description = "Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh/ps1` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" default = [] nullable = false } variable "after_cluster_joining_userdata" { type = list(string) - description = "Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" + description = <<-EOT + Additional (bash for linux, powershell for Windows) commands to execute on each worker node after joining the EKS cluster + (after executing the `bootstrap` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" + This can only be used with AL2 and Windows AMI types with a custom AMI set as EKS will inject a bootstrapper into all other user-scripts + EOT default = [] nullable = false }