updates for cirrus v2 compatibility

Author: jkeifer

CHANGELOG.md

@@ -15,6 +15,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Added SSM parameter for Cirrus process queue ARN (`CIRRUS_PROCESS_QUEUE_ARN`)
 
+- Added `payload_root_prefix` input variable to the cirrus module. This value
+  is propagated as the `CIRRUS_PAYLOAD_ROOT_PREFIX` environment variable to all
+  builtin Cirrus Lambda functions, stored as an SSM parameter, and available as
+  a builtin template variable in definition YAMLs. Defaults to `"cirrus"`.
+  Only functional with a cirrus version >=2.
+
+- Added `payload_tmp_lifecycle_expiration_days` input variable to the cirrus
+  module. When the payload bucket is created by the module, an S3 lifecycle
+  rule expires objects under `{payload_root_prefix}/tmp/` after the configured
+  number of days (default 10). Set to 0 to disable. Only functional with a
+  cirrus version >=2.
+
 ### Change
 
 ### Fixed
@@ -49,9 +61,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - TiTiler module will note three IAM related resources being created: aws_iam_role_policy.titiler-mosaic-lambda-inline-policy, aws_iam_role_policy_attachment.lambda_basic_execution, aws_iam_role_policy_attachment.lambda_vpc_access. This is a result of AWS provider deprecations to managed_policy and inline_policy properties. The TiTiler role in question will updated in place, resulting in no material changes to the role.
 
-- Removing internal stac-server terraform module code and pointed to new location as stac-server terraform module code now lives in it's own [stac-server terraform repo](https://github.com/Element84/terraform-aws-stac-server) ([216](https://github.com/Element84/filmdrop-aws-tf-modules/pull/216)). 
+- Removing internal stac-server terraform module code and pointed to new location as stac-server terraform module code now lives in it's own [stac-server terraform repo](https://github.com/Element84/terraform-aws-stac-server) ([216](https://github.com/Element84/filmdrop-aws-tf-modules/pull/216)).
 
-- Removing internal console-ui terraform module and pointing to an externally sourced module [filmdrop-ui](https://github.com/Element84/terraform-aws-filmdrop-ui) ([216](https://github.com/Element84/filmdrop-aws-tf-modules/pull/216)). 
+- Removing internal console-ui terraform module and pointing to an externally sourced module [filmdrop-ui](https://github.com/Element84/terraform-aws-filmdrop-ui) ([216](https://github.com/Element84/filmdrop-aws-tf-modules/pull/216)).
 
 - `cirrus api` lambda extracted into its own module, and can now be optionally deployed (or not) along with other cirrus built in lambdas when deploying the larger cirrus ecosystem ([248](https://github.com/Element84/filmdrop-aws-tf-modules/pull/248))
 

ci.tfvars

@@ -268,6 +268,8 @@ cirrus_inputs = {
   workflow_definitions_variables_ssm           = null
   workflow_metrics_cloudwatch_enabled          = true
   workflow_metrics_timestream_enabled          = true
+  payload_root_prefix                          = "cirrus"
+  payload_tmp_lifecycle_expiration_days         = 10
 }
 
 cirrus_dashboard_inputs = {

default.tfvars

@@ -279,6 +279,8 @@ cirrus_inputs = {
   workflow_definitions_variables_ssm           = null
   workflow_metrics_cloudwatch_enabled          = true
   workflow_metrics_timestream_enabled          = true
+  payload_root_prefix                          = "cirrus"
+  payload_tmp_lifecycle_expiration_days         = 10
 }
 
 cirrus_dashboard_inputs = {

inputs.tf

@@ -571,6 +571,8 @@ variable "cirrus_inputs" {
     cirrus_cli_iam_role_trust_principal          = optional(list(string))
     workflow_metrics_cloudwatch_enabled          = optional(bool)
     workflow_metrics_timestream_enabled          = optional(bool)
+    payload_root_prefix                          = optional(string, "cirrus")
+    payload_tmp_lifecycle_expiration_days         = optional(number, 10)
   })
   default = {
     data_bucket                               = "cirrus-data-bucket-name"
@@ -632,6 +634,8 @@ variable "cirrus_inputs" {
     cirrus_cli_iam_role_trust_principal          = null
     workflow_metrics_cloudwatch_enabled          = false
     workflow_metrics_timestream_enabled          = true
+    payload_root_prefix                          = "cirrus"
+    payload_tmp_lifecycle_expiration_days         = 10
   }
 }
 

modules/cirrus/README.md

@@ -15,6 +15,8 @@ module "base" {
   workflow_metrics_timestream_enabled                       = var.workflow_metrics_timestream_enabled
   cirrus_data_bucket                                        = var.cirrus_data_bucket
   cirrus_payload_bucket                                     = var.cirrus_payload_bucket
+  payload_root_prefix                                       = var.payload_root_prefix
+  payload_tmp_lifecycle_expiration_days                      = var.payload_tmp_lifecycle_expiration_days
   warning_sns_topic_arn                                     = var.warning_sns_topic_arn
   deploy_alarms                                             = var.deploy_alarms
 }

modules/cirrus/base.tf

@@ -45,6 +45,18 @@ variable "cirrus_payload_bucket" {
   type        = string
 }
 
+variable "payload_root_prefix" {
+  description = "Root prefix for payloads in the cirrus payload bucket."
+  type        = string
+  default     = "cirrus"
+}
+
+variable "payload_tmp_lifecycle_expiration_days" {
+  description = "Days after which objects under {payload_root_prefix}/tmp/ are expired. 0 disables."
+  type        = number
+  default     = 10
+}
+
 variable "warning_sns_topic_arn" {
   description = "String with FilmDrop Warning SNS topic ARN"
   type        = string

modules/cirrus/base/inputs.tf

@@ -11,3 +11,22 @@ resource "aws_s3_bucket" "cirrus_payload_bucket" {
   bucket_prefix = "${var.resource_prefix}-payload-"
   force_destroy = true
 }
+
+resource "aws_s3_bucket_lifecycle_configuration" "cirrus_payload_bucket" {
+  count = var.cirrus_payload_bucket == "" && var.payload_tmp_lifecycle_expiration_days > 0 ? 1 : 0
+
+  bucket = aws_s3_bucket.cirrus_payload_bucket[0].id
+
+  rule {
+    id     = "expire-tmp-objects"
+    status = "Enabled"
+
+    filter {
+      prefix = "${var.payload_root_prefix}/tmp/"
+    }
+
+    expiration {
+      days = var.payload_tmp_lifecycle_expiration_days
+    }
+  }
+}

modules/cirrus/base/s3.tf

@@ -9,6 +9,7 @@ module "api" {
   cirrus_log_level             = var.cirrus_log_level
   cirrus_data_bucket           = var.cirrus_data_bucket
   cirrus_payload_bucket        = var.cirrus_payload_bucket
+  payload_root_prefix          = var.payload_root_prefix
   cirrus_lambda_version        = var.cirrus_lambda_version
   cirrus_lambda_zip_filepath   = var.cirrus_lambda_zip_filepath
   cirrus_lambda_pyversion      = var.cirrus_lambda_pyversion

modules/cirrus/builtin-functions/api.tf

@@ -152,9 +152,10 @@ resource "aws_lambda_function" "cirrus_api" {
   environment {
     variables = merge(
       {
-        CIRRUS_LOG_LEVEL      = var.cirrus_log_level
-        CIRRUS_DATA_BUCKET    = var.cirrus_data_bucket
-        CIRRUS_PAYLOAD_BUCKET = var.cirrus_payload_bucket
+        CIRRUS_LOG_LEVEL           = var.cirrus_log_level
+        CIRRUS_DATA_BUCKET         = var.cirrus_data_bucket
+        CIRRUS_PAYLOAD_BUCKET      = var.cirrus_payload_bucket
+        CIRRUS_PAYLOAD_ROOT_PREFIX = var.payload_root_prefix
         CIRRUS_STATE_DB       = var.cirrus_state_dynamodb_table_name
       },
       var.workflow_metrics_timestream_enabled ? {