From 4a53ba545c84fedb7263ce2e482b3cc12bd9f221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emir=20=C3=96zbir?= Date: Sat, 22 Feb 2020 22:34:56 +0300 Subject: [PATCH 1/3] DMS module added --- dms/dms.tf | 44 +++++++++++++++++ dms/dms_endpoint.tf | 29 +++++++++++ dms/dms_task.tf | 13 +++++ dms/example/main.tf | 8 +++ dms/iam.tf | 54 +++++++++++++++++++++ dms/variables.tf | 115 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 263 insertions(+) create mode 100644 dms/dms.tf create mode 100644 dms/dms_endpoint.tf create mode 100644 dms/dms_task.tf create mode 100644 dms/example/main.tf create mode 100644 dms/iam.tf create mode 100644 dms/variables.tf diff --git a/dms/dms.tf b/dms/dms.tf new file mode 100644 index 0000000..fd2053a --- /dev/null +++ b/dms/dms.tf @@ -0,0 +1,44 @@ +data "aws_availability_zones" "available" { + state = "available" +} + + +resource "aws_dms_replication_instance" "dms_replication_instance" { + allocated_storage = var.allocated_storage + + apply_immediately = var.apply_immediately + auto_minor_version_upgrade = var.minor_version_upgrade + availability_zone = element(data.aws_availability_zones.available.names, 0) + engine_version = var.engine_version + kms_key_arn = aws_kms_key.dms_customer_key[0].arn + multi_az = var.is_multi_az + preferred_maintenance_window = var.maintaince_window + publicly_accessible = var.publicly_accessible + replication_instance_class = var.instance_class + replication_instance_id = var.instance_id + replication_subnet_group_id = aws_dms_replication_subnet_group.dms_subnet_group.id + + tags = var.tags + + vpc_security_group_ids = [ + for sec_group in var.security_groups : + sec_group + ] +} + +resource "aws_kms_key" "dms_customer_key" { + count = var.use_default_kms == true ? 1 : 0 + description = "DMS custom managed key" + deletion_window_in_days = var.deletion_window +} + + +resource "aws_dms_replication_subnet_group" "dms_subnet_group" { + replication_subnet_group_description = "Dms Replication Subnet Group" + replication_subnet_group_id = "dms-mongodb-dynamodb-subnet-group" + + subnet_ids = [ + for subnet_id in var.subnet_ids : + subnet_id + ] +} diff --git a/dms/dms_endpoint.tf b/dms/dms_endpoint.tf new file mode 100644 index 0000000..7115596 --- /dev/null +++ b/dms/dms_endpoint.tf @@ -0,0 +1,29 @@ +resource "aws_dms_endpoint" "mongodb_source" { + certificate_arn = "${var.mongodb_cert_arn}" + database_name = "${var.mongodb_database_name}" + endpoint_id = "${var.mongodb_endpoint_id}" + endpoint_type = "source" + engine_name = "mongodb" + extra_connection_attributes = var.extra_conn_attr_mongodb + kms_key_arn = aws_kms_key.dms_customer_key[0].arn + port = "${var.mongodb_port}" + server_name = "${var.mongodb_server_name}" + ssl_mode = "${var.ssl_mode}" + + username = "${var.username}" + password = "${var.password}" + +} + + +resource "aws_dms_endpoint" "dynamodb_target" { + certificate_arn = var.mongodb_cert_arn + endpoint_id = var.dynamodb_target_name + endpoint_type = "target" + engine_name = "dynamodb" + extra_connection_attributes = var.extra_conn_attr_dynamodb + kms_key_arn = aws_kms_key.dms_customer_key[0].arn + service_access_role = aws_iam_role.dms-access-for-endpoint.name + ssl_mode = "${var.ssl_mode}" + +} \ No newline at end of file diff --git a/dms/dms_task.tf b/dms/dms_task.tf new file mode 100644 index 0000000..a67f423 --- /dev/null +++ b/dms/dms_task.tf @@ -0,0 +1,13 @@ +resource "aws_dms_replication_task" "test" { + migration_type = var.migration_type + replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn + replication_task_id = var.replication_task_id + source_endpoint_arn = aws_dms_endpoint.mongodb_source.endpoint_arn + table_mappings = data.local_file.map_rule_path.content + + target_endpoint_arn = aws_dms_endpoint.dynamodb_target.endpoint_arn +} + +data "local_file" "map_rule_path" { + filename = var.map_rule_path +} \ No newline at end of file diff --git a/dms/example/main.tf b/dms/example/main.tf new file mode 100644 index 0000000..931976a --- /dev/null +++ b/dms/example/main.tf @@ -0,0 +1,8 @@ +module "dms-mongo" { + source = "../" + replication_task_id = "repltask" + map_rule_path = "${path.module}/config/map_rule.json" + mongodb_server_name = "ec2-34-253-225-6.eu-west-1.compute.amazonaws.com" + + +} diff --git a/dms/iam.tf b/dms/iam.tf new file mode 100644 index 0000000..cefc3f5 --- /dev/null +++ b/dms/iam.tf @@ -0,0 +1,54 @@ +resource "aws_iam_policy" "dms_access_policy" { + name = "DynamodbDMSPolicy" + path = "/" + description = "Dynamodb DMS Policy" + + policy = < Date: Sun, 23 Feb 2020 03:16:54 +0300 Subject: [PATCH 2/3] Endpoint updated --- dms/dms_endpoint.tf | 7 +++++-- dms/dms_task.tf | 11 ++++++++--- dms/example/main.tf | 8 -------- dms/variables.tf | 8 ++++++++ 4 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 dms/example/main.tf diff --git a/dms/dms_endpoint.tf b/dms/dms_endpoint.tf index 7115596..bc3d2d2 100644 --- a/dms/dms_endpoint.tf +++ b/dms/dms_endpoint.tf @@ -12,7 +12,10 @@ resource "aws_dms_endpoint" "mongodb_source" { username = "${var.username}" password = "${var.password}" - + + mongodb_settings { + extract_doc_id = "${var.extract_doc_id}" + } } @@ -23,7 +26,7 @@ resource "aws_dms_endpoint" "dynamodb_target" { engine_name = "dynamodb" extra_connection_attributes = var.extra_conn_attr_dynamodb kms_key_arn = aws_kms_key.dms_customer_key[0].arn - service_access_role = aws_iam_role.dms-access-for-endpoint.name + service_access_role = aws_iam_role.dms-access-for-endpoint.arn ssl_mode = "${var.ssl_mode}" } \ No newline at end of file diff --git a/dms/dms_task.tf b/dms/dms_task.tf index a67f423..e13e1c1 100644 --- a/dms/dms_task.tf +++ b/dms/dms_task.tf @@ -3,11 +3,16 @@ resource "aws_dms_replication_task" "test" { replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn replication_task_id = var.replication_task_id source_endpoint_arn = aws_dms_endpoint.mongodb_source.endpoint_arn - table_mappings = data.local_file.map_rule_path.content - + table_mappings = data.local_file.map_rule_content.content + replication_task_settings = data.local_file.repl_task_content.content target_endpoint_arn = aws_dms_endpoint.dynamodb_target.endpoint_arn } -data "local_file" "map_rule_path" { + +data "local_file" "map_rule_content" { filename = var.map_rule_path +} + +data "local_file" "repl_task_content" { + filename = var.repl_task_path } \ No newline at end of file diff --git a/dms/example/main.tf b/dms/example/main.tf deleted file mode 100644 index 931976a..0000000 --- a/dms/example/main.tf +++ /dev/null @@ -1,8 +0,0 @@ -module "dms-mongo" { - source = "../" - replication_task_id = "repltask" - map_rule_path = "${path.module}/config/map_rule.json" - mongodb_server_name = "ec2-34-253-225-6.eu-west-1.compute.amazonaws.com" - - -} diff --git a/dms/variables.tf b/dms/variables.tf index 51c6521..29b2a41 100644 --- a/dms/variables.tf +++ b/dms/variables.tf @@ -113,3 +113,11 @@ variable "map_rule_path" { variable "replication_task_id" { } + +variable "repl_task_path" { + +} + +variable "extract_doc_id" { + default = true +} From 03d927e65bbe5402fb162c132e1d234163ca83b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emir=20=C3=96zbir?= Date: Mon, 24 Feb 2020 00:26:10 +0300 Subject: [PATCH 3/3] updated endpoint for impl --- dms/dms_endpoint.tf | 22 ++++++++++++---------- dms/variables.tf | 9 +++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/dms/dms_endpoint.tf b/dms/dms_endpoint.tf index bc3d2d2..d107c9a 100644 --- a/dms/dms_endpoint.tf +++ b/dms/dms_endpoint.tf @@ -1,20 +1,22 @@ resource "aws_dms_endpoint" "mongodb_source" { - certificate_arn = "${var.mongodb_cert_arn}" - database_name = "${var.mongodb_database_name}" - endpoint_id = "${var.mongodb_endpoint_id}" + certificate_arn = var.mongodb_cert_arn + database_name = var.mongodb_database_name + endpoint_id = var.mongodb_endpoint_id endpoint_type = "source" engine_name = "mongodb" extra_connection_attributes = var.extra_conn_attr_mongodb kms_key_arn = aws_kms_key.dms_customer_key[0].arn - port = "${var.mongodb_port}" - server_name = "${var.mongodb_server_name}" - ssl_mode = "${var.ssl_mode}" + port = var.mongodb_port + server_name = var.mongodb_server_name + ssl_mode = var.ssl_mode - username = "${var.username}" - password = "${var.password}" + username = var.username + password = var.password mongodb_settings { - extract_doc_id = "${var.extract_doc_id}" + extract_doc_id = "${var.extract_doc_id}" + docs_to_investigate = 1000 + nesting_level = "${var.nesting_level}" } } @@ -27,6 +29,6 @@ resource "aws_dms_endpoint" "dynamodb_target" { extra_connection_attributes = var.extra_conn_attr_dynamodb kms_key_arn = aws_kms_key.dms_customer_key[0].arn service_access_role = aws_iam_role.dms-access-for-endpoint.arn - ssl_mode = "${var.ssl_mode}" + ssl_mode = var.ssl_mode } \ No newline at end of file diff --git a/dms/variables.tf b/dms/variables.tf index 29b2a41..31b8df1 100644 --- a/dms/variables.tf +++ b/dms/variables.tf @@ -121,3 +121,12 @@ variable "repl_task_path" { variable "extract_doc_id" { default = true } + +variable "metadata_mode" { + default = "document" +} + + +variable "nesting_level" { + default = "one" +}