From b4a4e396bf3dd397f27d3a0670fd94be3ac84b3f Mon Sep 17 00:00:00 2001 From: nikhil-goenka <70277861+nikhil-goenka@users.noreply.github.com> Date: Fri, 25 Sep 2020 12:00:49 +0530 Subject: [PATCH 1/4] f/aws_glue_crawler- allow connectionname for s3 target --- aws/resource_aws_glue_crawler.go | 9 +++++++++ website/docs/r/glue_crawler.html.markdown | 1 + 2 files changed, 10 insertions(+) diff --git a/aws/resource_aws_glue_crawler.go b/aws/resource_aws_glue_crawler.go index 418620f83b2..1785edbb9e0 100644 --- a/aws/resource_aws_glue_crawler.go +++ b/aws/resource_aws_glue_crawler.go @@ -105,6 +105,10 @@ func resourceAwsGlueCrawler() *schema.Resource { MinItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ + "connection_name": { + Type: schema.TypeString, + Required: true, + }, "path": { Type: schema.TypeString, Required: true, @@ -411,6 +415,10 @@ func expandGlueS3Target(cfg map[string]interface{}) *glue.S3Target { Path: aws.String(cfg["path"].(string)), } + if connection, ok := cfg["connection_name"]; ok { + target.ConnectionName = aws.String(connection.(string)) + } + if exclusions, ok := cfg["exclusions"]; ok { target.Exclusions = expandStringList(exclusions.([]interface{})) } @@ -604,6 +612,7 @@ func flattenGlueS3Targets(s3Targets []*glue.S3Target) []map[string]interface{} { attrs := make(map[string]interface{}) attrs["exclusions"] = flattenStringList(s3Target.Exclusions) attrs["path"] = aws.StringValue(s3Target.Path) + attrs["connection_name"] = aws.StringValue(s3Target.ConnectionName) result = append(result, attrs) } diff --git a/website/docs/r/glue_crawler.html.markdown b/website/docs/r/glue_crawler.html.markdown index ae45f8e8aa5..a9197c97b9a 100644 --- a/website/docs/r/glue_crawler.html.markdown +++ b/website/docs/r/glue_crawler.html.markdown @@ -120,6 +120,7 @@ The following arguments are supported: ### s3_target Argument Reference * `path` - (Required) The path to the Amazon S3 target. +* `connection_name` - (Optional) The name of a connection which allows crawler to access data in S3 within a VPC. * `exclusions` - (Optional) A list of glob patterns used to exclude from the crawl. ### catalog_target Argument Reference From abbada5d7e3f1bc64d5a7fff822ef4b8520f8232 Mon Sep 17 00:00:00 2001 From: nikhil-goenka <70277861+nikhil-goenka@users.noreply.github.com> Date: Fri, 25 Sep 2020 15:13:46 +0530 Subject: [PATCH 2/4] f/aws_glue_crawler:support for s3_target connection name --- aws/resource_aws_glue_crawler.go | 6 +- aws/resource_aws_glue_crawler_test.go | 107 ++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_glue_crawler.go b/aws/resource_aws_glue_crawler.go index 1785edbb9e0..bc79fa0fe4a 100644 --- a/aws/resource_aws_glue_crawler.go +++ b/aws/resource_aws_glue_crawler.go @@ -105,9 +105,9 @@ func resourceAwsGlueCrawler() *schema.Resource { MinItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "connection_name": { + "connection_name": { Type: schema.TypeString, - Required: true, + Optional: true, }, "path": { Type: schema.TypeString, @@ -415,7 +415,7 @@ func expandGlueS3Target(cfg map[string]interface{}) *glue.S3Target { Path: aws.String(cfg["path"].(string)), } - if connection, ok := cfg["connection_name"]; ok { + if connection, ok := cfg["connection_name"]; ok { target.ConnectionName = aws.String(connection.(string)) } diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 6d93e9ffad0..55b31702640 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -446,6 +446,35 @@ func TestAccAWSGlueCrawler_S3Target(t *testing.T) { }) } +func TestAccAWSGlueCrawler_S3Target_ConnectionName(t *testing.T) { + var crawler glue.Crawler + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_crawler.test" + connectionName := "aws_glue_connection.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueCrawlerDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCrawlerConfig_S3Target_ConnectionName(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("crawler/%s", rName)), + resource.TestCheckResourceAttr(resourceName, "s3_target.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.connection_name", connectionName, "name"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccAWSGlueCrawler_S3Target_Exclusions(t *testing.T) { var crawler glue.Crawler rName := acctest.RandomWithPrefix("tf-acc-test") @@ -1679,6 +1708,84 @@ resource "aws_glue_crawler" "test" { `, rName, rName, exclusion1) } +func testAccGlueCrawlerConfig_S3Target_ConnectionName(rName string) string { + return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(` +data "aws_availability_zones" "available" { + state = "available" + + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + + tags = { + Name = "terraform-testacc-glue-connection-base" + } +} + +resource "aws_security_group" "test" { + name = "%[1]s" + vpc_id = aws_vpc.test.id + + ingress { + from_port = 1 + protocol = "tcp" + self = true + to_port = 65535 + } +} + +resource "aws_subnet" "test" { + count = 2 + + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = "10.0.${count.index}.0/24" + vpc_id = aws_vpc.test.id + + tags = { + Name = "terraform-testacc-glue-connection-base" + } +} + +resource "aws_glue_catalog_database" "test" { + name = "%[1]s" +} + +resource "aws_glue_connection" "test" { + connection_properties = { + JDBC_ENFORCE_SSL = false + } + + connection_type = "NETWORK" + + name = "%s" + + physical_connection_requirements { + availability_zone = aws_subnet.test[0].availability_zone + security_group_id_list = [aws_security_group.test.id] + subnet_id = aws_subnet.test[0].id + } +} + +resource "aws_glue_crawler" "test" { + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] + + database_name = aws_glue_catalog_database.test.name + name = "%[1]s" + role = aws_iam_role.test.name + + s3_target { + connection_name = aws_glue_connection.test.name + path = "s3://bucket1" + } +} +`, rName, rName) +} + func testAccGlueCrawlerConfig_S3Target_Exclusions2(rName, exclusion1, exclusion2 string) string { return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { From da4b4bfc5d96305f55e8e1e692780b8e93d725ee Mon Sep 17 00:00:00 2001 From: nikhil-goenka <70277861+nikhil-goenka@users.noreply.github.com> Date: Fri, 25 Sep 2020 17:07:23 +0530 Subject: [PATCH 3/4] f/aws_glue_crawler:support for s3_target connection_name --- aws/resource_aws_glue_crawler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 55b31702640..71155f21179 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -1762,7 +1762,7 @@ resource "aws_glue_connection" "test" { connection_type = "NETWORK" - name = "%s" + name = "%[1]s" physical_connection_requirements { availability_zone = aws_subnet.test[0].availability_zone @@ -1783,7 +1783,7 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket1" } } -`, rName, rName) +`, rName) } func testAccGlueCrawlerConfig_S3Target_Exclusions2(rName, exclusion1, exclusion2 string) string { From 622f46344539f36946fdc59adef7d23ae4a2f4e9 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Fri, 25 Sep 2020 12:57:37 -0700 Subject: [PATCH 4/4] Terraform formatting fix --- aws/resource_aws_glue_crawler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 71155f21179..f8d6536c2c8 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -1780,7 +1780,7 @@ resource "aws_glue_crawler" "test" { s3_target { connection_name = aws_glue_connection.test.name - path = "s3://bucket1" + path = "s3://bucket1" } } `, rName)