diff --git a/streaming/kafka/README.md b/streaming/kafka/README.md index 4a8af9af4..ffaa1d394 100644 --- a/streaming/kafka/README.md +++ b/streaming/kafka/README.md @@ -8,30 +8,29 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | ~> 5.95 | -| [helm](#requirement\_helm) | ~> 2.17 | -| [kubectl](#requirement\_kubectl) | >= 2.0.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [random](#requirement\_random) | >= 3.1 | +| [aws](#requirement\_aws) | ~> 6.0 | +| [helm](#requirement\_helm) | ~> 2.9 | +| [kubectl](#requirement\_kubectl) | ~> 2.1 | +| [kubernetes](#requirement\_kubernetes) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.8 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | ~> 5.95 | -| [aws.virginia](#provider\_aws.virginia) | ~> 5.95 | -| [kubernetes](#provider\_kubernetes) | >= 2.10 | -| [random](#provider\_random) | >= 3.1 | +| [aws](#provider\_aws) | ~> 6.0 | +| [aws.virginia](#provider\_aws.virginia) | ~> 6.0 | +| [kubernetes](#provider\_kubernetes) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.8 | ## Modules | Name | Source | Version | |------|--------|---------| | [amp\_ingest\_irsa](#module\_amp\_ingest\_irsa) | aws-ia/eks-blueprints-addon/aws | ~> 1.0 | -| [aws\_auth](#module\_aws\_auth) | terraform-aws-modules/eks/aws//modules/aws-auth | 20.17.2 | -| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | 20.17.2 | -| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | +| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts | ~> 6.4 | +| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 21.0 | +| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.23 | | [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | 1.35.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | @@ -39,12 +38,13 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | Name | Type | |------|------| +| [aws_eks_access_entry.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_iam_policy.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | | [aws_secretsmanager_secret.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | | [aws_secretsmanager_secret_version.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | | [kubernetes_annotations.gp2_default](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource | -| [kubernetes_storage_class.ebs_csi_encrypted_gp3_storage_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class) | resource | +| [kubernetes_storage_class_v1.ebs_csi_encrypted_gp3_storage_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | | [random_password.grafana](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | @@ -58,7 +58,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.33"` | no | +| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.35"` | no | | [eks\_data\_plane\_subnet\_secondary\_cidr](#input\_eks\_data\_plane\_subnet\_secondary\_cidr) | Secondary CIDR blocks. 32766 IPs per Subnet per Subnet/AZ for EKS Node and Pods | `list(string)` |
[
"100.64.0.0/17",
"100.64.128.0/17"
]
| no | | [enable\_amazon\_prometheus](#input\_enable\_amazon\_prometheus) | Enable AWS Managed Prometheus service | `bool` | `true` | no | | [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"kafka-on-eks"` | no | diff --git a/streaming/kafka/addons.tf b/streaming/kafka/addons.tf index 8823e6668..22adf84d3 100644 --- a/streaming/kafka/addons.tf +++ b/streaming/kafka/addons.tf @@ -15,7 +15,7 @@ resource "kubernetes_annotations" "gp2_default" { depends_on = [module.eks] } -resource "kubernetes_storage_class" "ebs_csi_encrypted_gp3_storage_class" { +resource "kubernetes_storage_class_v1" "ebs_csi_encrypted_gp3_storage_class" { metadata { name = "gp3" annotations = { @@ -41,15 +41,16 @@ resource "kubernetes_storage_class" "ebs_csi_encrypted_gp3_storage_class" { #--------------------------------------------------------------- module "ebs_csi_driver_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.20" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "~> 6.4" - role_name_prefix = "${module.eks.cluster_name}-ebs-csi-driver-" + name = "${module.eks.cluster_name}-ebs-csi-driver-" + use_name_prefix = true attach_ebs_csi_policy = true oidc_providers = { - main = { + this = { provider_arn = module.eks.oidc_provider_arn namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] } @@ -63,7 +64,7 @@ module "ebs_csi_driver_irsa" { #--------------------------------------------------------------- module "eks_blueprints_addons" { source = "aws-ia/eks-blueprints-addons/aws" - version = "~> 1.2" + version = "~> 1.23" cluster_name = module.eks.cluster_name cluster_endpoint = module.eks.cluster_endpoint @@ -75,16 +76,7 @@ module "eks_blueprints_addons" { #--------------------------------------- eks_addons = { aws-ebs-csi-driver = { - service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn - } - coredns = { - preserve = true - } - vpc-cni = { - preserve = true - } - kube-proxy = { - preserve = true + service_account_role_arn = module.ebs_csi_driver_irsa.arn } } @@ -108,7 +100,7 @@ module "eks_blueprints_addons" { #--------------------------------------- enable_karpenter = true karpenter = { - chart_version = "1.6.2" # Compatible with Kubernetes 1.33 + chart_version = "1.8.6" # Latest version compatible with Kubernetes 1.35 repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password timeout = 600 # 10 minutes @@ -143,7 +135,7 @@ module "eks_blueprints_addons" { amp_irsa = module.amp_ingest_irsa[0].iam_role_arn amp_remotewrite_url = "https://aps-workspaces.${local.region}.amazonaws.com/workspaces/${aws_prometheus_workspace.amp[0].id}/api/v1/remote_write" amp_url = "https://aps-workspaces.${local.region}.amazonaws.com/workspaces/${aws_prometheus_workspace.amp[0].id}" - storage_class_type = kubernetes_storage_class.ebs_csi_encrypted_gp3_storage_class.id + storage_class_type = kubernetes_storage_class_v1.ebs_csi_encrypted_gp3_storage_class.id }) : templatefile("${path.module}/helm-values/kube-prometheus.yaml", {}) ] chart_version = "48.1.1" @@ -194,7 +186,7 @@ resource "aws_secretsmanager_secret_version" "grafana" { #--------------------------------------------------------------- module "eks_data_addons" { source = "aws-ia/eks-data-addons/aws" - version = "1.35.0" # Updated for better Kubernetes 1.33 support + version = "1.35.0" # Updated for better Kubernetes 1.35 support oidc_provider_arn = module.eks.oidc_provider_arn #--------------------------------------------------------------- @@ -206,7 +198,7 @@ module "eks_data_addons" { operating_system = "linux" node_group_type = "core" })], - version = "0.46.0" # Latest version with EKS 1.33 compatibility + version = "0.50.0" # Latest version with EKS 1.35 compatibility timeout = 900 # 15 minutes wait = true wait_for_jobs = true diff --git a/streaming/kafka/eks.tf b/streaming/kafka/eks.tf index f2c3fca36..97d2bf056 100644 --- a/streaming/kafka/eks.tf +++ b/streaming/kafka/eks.tf @@ -3,13 +3,13 @@ #--------------------------------------------------------------- module "eks" { source = "terraform-aws-modules/eks/aws" - version = "20.17.2" + version = "~> 21.0" - cluster_name = local.name - cluster_version = local.cluster_version + name = local.name + kubernetes_version = local.cluster_version #WARNING: Avoid using this option (cluster_endpoint_public_access = true) in preprod or prod accounts. This feature is designed for sandbox accounts, simplifying cluster deployment and testing. - cluster_endpoint_public_access = true + endpoint_public_access = true vpc_id = module.vpc.vpc_id # Filtering only Secondary CIDR private subnets starting with "100.". Subnet IDs where the EKS Control Plane ENIs will be created @@ -21,11 +21,22 @@ module "eks" { authentication_mode = "API_AND_CONFIG_MAP" enable_cluster_creator_admin_permissions = true + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } + } + #--------------------------------------- # Note: This can further restricted to specific required for each Add-on and your application #--------------------------------------- # Extend cluster security group rules - cluster_security_group_additional_rules = { + security_group_additional_rules = { ingress_nodes_ephemeral_ports_tcp = { description = "Nodes on ephemeral ports" protocol = "tcp" @@ -95,20 +106,8 @@ module "eks" { }) } -module "aws_auth" { - source = "terraform-aws-modules/eks/aws//modules/aws-auth" - version = "20.17.2" - - manage_aws_auth_configmap = true - - aws_auth_roles = [ - { - rolearn = module.eks_blueprints_addons.karpenter.node_iam_role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - } - ] +resource "aws_eks_access_entry" "karpenter" { + cluster_name = module.eks.cluster_name + principal_arn = module.eks_blueprints_addons.karpenter.node_iam_role_arn + type = "EC2_LINUX" } diff --git a/streaming/kafka/helm-values/strimzi-kafka-values.yaml b/streaming/kafka/helm-values/strimzi-kafka-values.yaml index 52cee47eb..6427562db 100644 --- a/streaming/kafka/helm-values/strimzi-kafka-values.yaml +++ b/streaming/kafka/helm-values/strimzi-kafka-values.yaml @@ -10,7 +10,7 @@ watchAnyNamespace: true defaultImageRegistry: quay.io defaultImageRepository: strimzi -defaultImageTag: 0.46.0 +defaultImageTag: 0.50.0 nodeSelector: kubernetes.io/os: ${operating_system} diff --git a/streaming/kafka/helper.sh b/streaming/kafka/helper.sh index 1f710bd4b..7be78f618 100644 --- a/streaming/kafka/helper.sh +++ b/streaming/kafka/helper.sh @@ -157,7 +157,7 @@ case "$1" in --bootstrap-server cluster-kafka-bootstrap:9092 ;; send-messages-to-kafka-failover-topic-from-producer) - kubectl -n kafka run kafka-producer -ti --image=quay.io/strimzi/kafka:0.43.0-kafka-3.8.0 --rm=true --restart=Never -- bin/kafka-console-producer.sh --bootstrap-server cluster-kafka-bootstrap:9092 --topic test-topic-failover + kubectl -n kafka run kafka-producer -ti --image=quay.io/strimzi/kafka:0.50.0-kafka-4.1.1 --rm=true --restart=Never -- bin/kafka-console-producer.sh --bootstrap-server cluster-kafka-bootstrap:9092 --topic test-topic-failover ;; read-messages-from-kafka-failover-topic-consumer) kubectl exec -it kafka-cli -n kafka -- bin/kafka-console-consumer.sh \ diff --git a/streaming/kafka/kafka-manifests/kafka-cluster.yaml b/streaming/kafka/kafka-manifests/kafka-cluster.yaml index e43d3e165..01a7b5b00 100644 --- a/streaming/kafka/kafka-manifests/kafka-cluster.yaml +++ b/streaming/kafka/kafka-manifests/kafka-cluster.yaml @@ -1,4 +1,4 @@ -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: Kafka metadata: name: cluster @@ -8,8 +8,8 @@ metadata: strimzi.io/node-pools: enabled spec: kafka: - version: 3.9.0 - metadataVersion: 3.9-IV0 + version: 4.1.1 + metadataVersion: 4.1-IV0 listeners: - name: plain port: 9092 @@ -25,24 +25,9 @@ spec: transaction.state.log.min.isr: 2 default.replication.factor: 3 min.insync.replicas: 2 - resources: - requests: - memory: 58Gi - cpu: "6" - limits: - memory: 64Gi - cpu: "8" jvmOptions: "-Xmx": "6g" "-Xms": "4g" - storage: - type: jbod - volumes: - - id: 0 - type: persistent-claim - size: 1000Gi - class: gp3 - deleteClaim: false template: pod: tolerations: @@ -267,7 +252,7 @@ data: name: kafka_cruisecontrol_$1_$2 type: GAUGE --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: name: controller @@ -284,10 +269,11 @@ spec: - id: 0 type: persistent-claim size: 100Gi + class: gp3 kraftMetadata: shared deleteClaim: false --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: name: broker @@ -304,5 +290,13 @@ spec: - id: 0 type: persistent-claim size: 100Gi + class: gp3 kraftMetadata: shared deleteClaim: false + resources: + requests: + memory: 58Gi + cpu: "6" + limits: + memory: 64Gi + cpu: "8" diff --git a/streaming/kafka/kafka-manifests/kafka-rebalance.yaml b/streaming/kafka/kafka-manifests/kafka-rebalance.yaml index f1bb6cce9..29d9b38ec 100644 --- a/streaming/kafka/kafka-manifests/kafka-rebalance.yaml +++ b/streaming/kafka/kafka-manifests/kafka-rebalance.yaml @@ -1,4 +1,4 @@ -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaRebalance metadata: name: my-rebalance diff --git a/streaming/kafka/variables.tf b/streaming/kafka/variables.tf index 90fea665b..ce959b562 100644 --- a/streaming/kafka/variables.tf +++ b/streaming/kafka/variables.tf @@ -13,7 +13,7 @@ variable "name" { variable "eks_cluster_version" { description = "EKS Cluster version" type = string - default = "1.33" + default = "1.35" } # VPC diff --git a/streaming/kafka/versions.tf b/streaming/kafka/versions.tf index 45232641b..cf96a949e 100644 --- a/streaming/kafka/versions.tf +++ b/streaming/kafka/versions.tf @@ -4,23 +4,23 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.95" + version = "~> 6.0" } kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.10" + version = "~> 3.0" } helm = { source = "hashicorp/helm" - version = "~> 2.17" + version = "~> 2.9" # upgrading to helm v3 depends on https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/issues/452 } kubectl = { source = "alekc/kubectl" - version = ">= 2.0.2" + version = "~> 2.1" } random = { source = "hashicorp/random" - version = ">= 3.1" + version = "~> 3.8" } } }