diff --git a/modules/aws-glue/main.tf b/modules/aws-glue/main.tf index f982352..8dbe214 100644 --- a/modules/aws-glue/main.tf +++ b/modules/aws-glue/main.tf @@ -1,8 +1,6 @@ -module "glue_catalog_database" { - source = "terraform-aws-modules/glue/aws//modules/catalog-database" - version = "~> 1.0" +resource "aws_glue_catalog_database" "this" { + count = var.create_database ? 1 : 0 - create = var.create_database name = var.database_name description = var.database_description catalog_id = var.catalog_id @@ -12,13 +10,11 @@ module "glue_catalog_database" { tags = var.tags } -module "glue_crawler" { - source = "terraform-aws-modules/glue/aws//modules/crawler" - version = "~> 1.0" +resource "aws_glue_crawler" "this" { + count = var.create_crawler ? 1 : 0 - create = var.create_crawler name = var.crawler_name - database_name = var.create_database ? module.glue_catalog_database.name : var.database_name + database_name = var.create_database ? aws_glue_catalog_database.this[0].name : var.database_name role = var.crawler_role_arn description = var.crawler_description schedule = var.crawler_schedule @@ -26,11 +22,28 @@ module "glue_crawler" { classifiers = var.crawler_classifiers configuration = var.crawler_configuration - s3_targets = var.s3_targets - schema_change_policy = var.schema_change_policy - recrawl_policy = var.recrawl_policy + dynamic "s3_target" { + for_each = var.s3_targets + content { + path = s3_target.value.path + exclusions = try(s3_target.value.exclusions, []) + connection_name = s3_target.value.connection_name + event_queue_arn = s3_target.value.event_queue_arn + dlq_event_queue_arn = s3_target.value.dlq_event_queue_arn + sample_size = s3_target.value.sample_size + } + } + + schema_change_policy { + update_behavior = try(var.schema_change_policy.update_behavior, "UPDATE_IN_DATABASE") + delete_behavior = try(var.schema_change_policy.delete_behavior, "LOG") + } + + recrawl_policy { + recrawl_behavior = try(var.recrawl_policy.recrawl_behavior, "CRAWL_EVERYTHING") + } tags = var.tags - depends_on = [module.glue_catalog_database] + depends_on = [aws_glue_catalog_database.this] } diff --git a/modules/aws-glue/outputs.tf b/modules/aws-glue/outputs.tf index 1f51953..717c4a1 100644 --- a/modules/aws-glue/outputs.tf +++ b/modules/aws-glue/outputs.tf @@ -1,24 +1,24 @@ output "database_name" { description = "Name of the Glue catalog database." - value = var.create_database ? module.glue_catalog_database.name : var.database_name + value = var.create_database ? aws_glue_catalog_database.this[0].name : var.database_name } output "database_id" { description = "ID of the Glue catalog database (catalog_id:name)." - value = var.create_database ? module.glue_catalog_database.id : null + value = var.create_database ? aws_glue_catalog_database.this[0].id : null } output "crawler_id" { description = "ID of the Glue crawler (same as its name)." - value = var.create_crawler ? module.glue_crawler.id : null + value = var.create_crawler ? aws_glue_crawler.this[0].id : null } output "crawler_arn" { description = "ARN of the Glue crawler." - value = var.create_crawler ? module.glue_crawler.arn : null + value = var.create_crawler ? aws_glue_crawler.this[0].arn : null } output "crawler_name" { description = "Name of the Glue crawler." - value = var.create_crawler ? module.glue_crawler.name : null + value = var.create_crawler ? aws_glue_crawler.this[0].id : null } diff --git a/modules/aws-glue/variables.tf b/modules/aws-glue/variables.tf index f493dda..b118a55 100644 --- a/modules/aws-glue/variables.tf +++ b/modules/aws-glue/variables.tf @@ -93,7 +93,7 @@ variable "s3_targets" { description = "List of S3 target configurations for the crawler." type = list(object({ path = string - exclusions = optional(list(string), []) + exclusions = optional(list(string)) connection_name = optional(string) event_queue_arn = optional(string) dlq_event_queue_arn = optional(string) @@ -105,8 +105,8 @@ variable "s3_targets" { variable "schema_change_policy" { description = "Behavior when the crawler discovers a changed schema. update_behavior: UPDATE_IN_DATABASE or LOG. delete_behavior: DELETE_FROM_DATABASE, LOG, or DEPRECATE_IN_DATABASE." type = object({ - update_behavior = optional(string, "UPDATE_IN_DATABASE") - delete_behavior = optional(string, "LOG") + update_behavior = optional(string) + delete_behavior = optional(string) }) default = {} } @@ -114,7 +114,7 @@ variable "schema_change_policy" { variable "recrawl_policy" { description = "Recrawl behavior. recrawl_behavior: CRAWL_EVERYTHING, CRAWL_NEW_FOLDERS_ONLY, or CRAWL_EVENT_MODE." type = object({ - recrawl_behavior = optional(string, "CRAWL_EVERYTHING") + recrawl_behavior = optional(string) }) default = {} }