diff --git a/aws/resource_aws_glue_catalog_table.go b/aws/resource_aws_glue_catalog_table.go index e803e449255..bd799eeae63 100644 --- a/aws/resource_aws_glue_catalog_table.go +++ b/aws/resource_aws_glue_catalog_table.go @@ -3,6 +3,7 @@ package aws import ( "fmt" "log" + "regexp" "strings" "github.com/aws/aws-sdk-go/aws" @@ -39,13 +40,18 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Required: true, }, "description": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 2048), }, "name": { Type: schema.TypeString, ForceNew: true, Required: true, + ValidateFunc: validation.All( + validation.StringLenBetween(1, 255), + validation.StringDoesNotMatch(regexp.MustCompile(`[A-Z]`), "uppercase characters cannot be used"), + ), }, "owner": { Type: schema.TypeString, @@ -62,23 +68,27 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "comment": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 255), }, "name": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), }, "type": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 131072), }, }, }, }, "retention": { - Type: schema.TypeInt, - Optional: true, + Type: schema.TypeInt, + Optional: true, + ValidateFunc: validation.IntAtLeast(0), }, "storage_descriptor": { Type: schema.TypeList, @@ -89,7 +99,10 @@ func resourceAwsGlueCatalogTable() *schema.Resource { "bucket_columns": { Type: schema.TypeList, Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringLenBetween(1, 255), + }, }, "columns": { Type: schema.TypeList, @@ -97,16 +110,24 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "comment": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 255), }, "name": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), }, - "type": { - Type: schema.TypeString, + "parameters": { + Type: schema.TypeMap, Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "type": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 131072), }, }, }, @@ -145,7 +166,7 @@ func resourceAwsGlueCatalogTable() *schema.Resource { "name": { Type: schema.TypeString, Optional: true, - ValidateFunc: validation.StringIsNotEmpty, + ValidateFunc: validation.StringLenBetween(1, 255), }, "parameters": { Type: schema.TypeMap, @@ -169,7 +190,10 @@ func resourceAwsGlueCatalogTable() *schema.Resource { "skewed_column_names": { Type: schema.TypeList, Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringLenBetween(1, 255), + }, }, "skewed_column_values": { Type: schema.TypeList, @@ -190,12 +214,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "column": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), }, "sort_order": { - Type: schema.TypeInt, - Required: true, + Type: schema.TypeInt, + Required: true, + ValidateFunc: validation.IntInSlice([]int{0, 1}), }, }, }, @@ -212,12 +238,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Optional: true, }, "view_original_text": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 409600), }, "view_expanded_text": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 409600), }, }, } @@ -243,9 +271,10 @@ func resourceAwsGlueCatalogTableCreate(d *schema.ResourceData, meta interface{}) TableInput: expandGlueTableInput(d), } + log.Printf("[DEBUG] Glue catalog table input: %#v", input) _, err := conn.CreateTable(input) if err != nil { - return fmt.Errorf("Error creating Catalog Table: %s", err) + return fmt.Errorf("Error creating Glue Catalog Table: %w", err) } d.SetId(fmt.Sprintf("%s:%s:%s", catalogID, dbName, name)) @@ -276,39 +305,40 @@ func resourceAwsGlueCatalogTableRead(d *schema.ResourceData, meta interface{}) e return nil } - return fmt.Errorf("Error reading Glue Catalog Table: %s", err) + return fmt.Errorf("Error reading Glue Catalog Table: %w", err) } + table := out.Table tableArn := arn.ARN{ Partition: meta.(*AWSClient).partition, Service: "glue", Region: meta.(*AWSClient).region, AccountID: meta.(*AWSClient).accountid, - Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(out.Table.Name)), + Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(table.Name)), }.String() d.Set("arn", tableArn) - d.Set("name", out.Table.Name) + d.Set("name", table.Name) d.Set("catalog_id", catalogID) d.Set("database_name", dbName) - d.Set("description", out.Table.Description) - d.Set("owner", out.Table.Owner) - d.Set("retention", out.Table.Retention) + d.Set("description", table.Description) + d.Set("owner", table.Owner) + d.Set("retention", table.Retention) - if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(out.Table.StorageDescriptor)); err != nil { - return fmt.Errorf("error setting storage_descriptor: %s", err) + if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(table.StorageDescriptor)); err != nil { + return fmt.Errorf("error setting storage_descriptor: %w", err) } - if err := d.Set("partition_keys", flattenGlueColumns(out.Table.PartitionKeys)); err != nil { - return fmt.Errorf("error setting partition_keys: %s", err) + if err := d.Set("partition_keys", flattenGlueColumns(table.PartitionKeys)); err != nil { + return fmt.Errorf("error setting partition_keys: %w", err) } - d.Set("view_original_text", out.Table.ViewOriginalText) - d.Set("view_expanded_text", out.Table.ViewExpandedText) - d.Set("table_type", out.Table.TableType) + d.Set("view_original_text", table.ViewOriginalText) + d.Set("view_expanded_text", table.ViewExpandedText) + d.Set("table_type", table.TableType) - if err := d.Set("parameters", aws.StringValueMap(out.Table.Parameters)); err != nil { - return fmt.Errorf("error setting parameters: %s", err) + if err := d.Set("parameters", aws.StringValueMap(table.Parameters)); err != nil { + return fmt.Errorf("error setting parameters: %w", err) } return nil @@ -329,7 +359,7 @@ func resourceAwsGlueCatalogTableUpdate(d *schema.ResourceData, meta interface{}) } if _, err := conn.UpdateTable(updateTableInput); err != nil { - return fmt.Errorf("Error updating Glue Catalog Table: %s", err) + return fmt.Errorf("Error updating Glue Catalog Table: %w", err) } return resourceAwsGlueCatalogTableRead(d, meta) @@ -350,7 +380,7 @@ func resourceAwsGlueCatalogTableDelete(d *schema.ResourceData, meta interface{}) DatabaseName: aws.String(dbName), }) if err != nil { - return fmt.Errorf("Error deleting Glue Catalog Table: %s", err.Error()) + return fmt.Errorf("Error deleting Glue Catalog Table: %w", err) } return nil } @@ -377,8 +407,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput { } if v, ok := d.GetOk("partition_keys"); ok { - columns := expandGlueColumns(v.([]interface{})) - tableInput.PartitionKeys = columns + tableInput.PartitionKeys = expandGlueColumns(v.([]interface{})) } if v, ok := d.GetOk("view_original_text"); ok { @@ -394,11 +423,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput { } if v, ok := d.GetOk("parameters"); ok { - paramsMap := map[string]string{} - for key, value := range v.(map[string]interface{}) { - paramsMap[key] = value.(string) - } - tableInput.Parameters = aws.StringMap(paramsMap) + tableInput.Parameters = stringMapToPointers(v.(map[string]interface{})) } return tableInput @@ -413,8 +438,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor { storageDescriptor := &glue.StorageDescriptor{} if v, ok := s["columns"]; ok { - columns := expandGlueColumns(v.([]interface{})) - storageDescriptor.Columns = columns + storageDescriptor.Columns = expandGlueColumns(v.([]interface{})) } if v, ok := s["location"]; ok { @@ -442,11 +466,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor { } if v, ok := s["bucket_columns"]; ok { - bucketColumns := make([]string, len(v.([]interface{}))) - for i, item := range v.([]interface{}) { - bucketColumns[i] = fmt.Sprint(item) - } - storageDescriptor.BucketColumns = aws.StringSlice(bucketColumns) + storageDescriptor.BucketColumns = expandStringList(v.([]interface{})) } if v, ok := s["sort_columns"]; ok { @@ -458,11 +478,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor { } if v, ok := s["parameters"]; ok { - paramsMap := map[string]string{} - for key, value := range v.(map[string]interface{}) { - paramsMap[key] = value.(string) - } - storageDescriptor.Parameters = aws.StringMap(paramsMap) + storageDescriptor.Parameters = stringMapToPointers(v.(map[string]interface{})) } if v, ok := s["stored_as_sub_directories"]; ok { @@ -489,6 +505,10 @@ func expandGlueColumns(columns []interface{}) []*glue.Column { column.Type = aws.String(v.(string)) } + if v, ok := elementMap["parameters"]; ok { + column.Parameters = stringMapToPointers(v.(map[string]interface{})) + } + columnSlice = append(columnSlice, column) } @@ -508,11 +528,7 @@ func expandGlueSerDeInfo(l []interface{}) *glue.SerDeInfo { } if v := s["parameters"]; len(v.(map[string]interface{})) > 0 { - paramsMap := map[string]string{} - for key, value := range v.(map[string]interface{}) { - paramsMap[key] = value.(string) - } - serDeInfo.Parameters = aws.StringMap(paramsMap) + serDeInfo.Parameters = stringMapToPointers(v.(map[string]interface{})) } if v := s["serialization_library"]; len(v.(string)) > 0 { @@ -551,27 +567,15 @@ func expandGlueSkewedInfo(l []interface{}) *glue.SkewedInfo { skewedInfo := &glue.SkewedInfo{} if v, ok := s["skewed_column_names"]; ok { - columnsSlice := make([]string, len(v.([]interface{}))) - for i, item := range v.([]interface{}) { - columnsSlice[i] = fmt.Sprint(item) - } - skewedInfo.SkewedColumnNames = aws.StringSlice(columnsSlice) + skewedInfo.SkewedColumnNames = expandStringList(v.([]interface{})) } if v, ok := s["skewed_column_value_location_maps"]; ok { - typeMap := map[string]string{} - for key, value := range v.(map[string]interface{}) { - typeMap[key] = value.(string) - } - skewedInfo.SkewedColumnValueLocationMaps = aws.StringMap(typeMap) + skewedInfo.SkewedColumnValueLocationMaps = stringMapToPointers(v.(map[string]interface{})) } if v, ok := s["skewed_column_values"]; ok { - columnsSlice := make([]string, len(v.([]interface{}))) - for i, item := range v.([]interface{}) { - columnsSlice[i] = fmt.Sprint(item) - } - skewedInfo.SkewedColumnValues = aws.StringSlice(columnsSlice) + skewedInfo.SkewedColumnValues = expandStringList(v.([]interface{})) } return skewedInfo @@ -605,8 +609,8 @@ func flattenGlueStorageDescriptor(s *glue.StorageDescriptor) []map[string]interf return storageDescriptors } -func flattenGlueColumns(cs []*glue.Column) []map[string]string { - columnsSlice := make([]map[string]string, len(cs)) +func flattenGlueColumns(cs []*glue.Column) []map[string]interface{} { + columnsSlice := make([]map[string]interface{}, len(cs)) if len(cs) > 0 { for i, v := range cs { columnsSlice[i] = flattenGlueColumn(v) @@ -616,8 +620,8 @@ func flattenGlueColumns(cs []*glue.Column) []map[string]string { return columnsSlice } -func flattenGlueColumn(c *glue.Column) map[string]string { - column := make(map[string]string) +func flattenGlueColumn(c *glue.Column) map[string]interface{} { + column := make(map[string]interface{}) if c == nil { return column @@ -635,6 +639,10 @@ func flattenGlueColumn(c *glue.Column) map[string]string { column["comment"] = v } + if v := c.Parameters; v != nil { + column["parameters"] = aws.StringValueMap(v) + } + return column } diff --git a/aws/resource_aws_glue_catalog_table_test.go b/aws/resource_aws_glue_catalog_table_test.go index 12914ad5baf..2f0427348db 100644 --- a/aws/resource_aws_glue_catalog_table_test.go +++ b/aws/resource_aws_glue_catalog_table_test.go @@ -63,6 +63,36 @@ func TestAccAWSGlueCatalogTable_basic(t *testing.T) { testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("table/%s/%s", rName, rName)), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttr(resourceName, "database_name", rName), + testAccCheckResourceAttrAccountID(resourceName, "catalog_id"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccAWSGlueCatalogTable_columnParameters(t *testing.T) { + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_catalog_table.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckGlueTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCatalogTableColumnParameters(rName), + Destroy: false, + Check: resource.ComposeTestCheckFunc( + testAccCheckGlueCatalogTableExists(resourceName), + resource.TestCheckResourceAttr(resourceName, "storage_descriptor.0.columns.#", "1"), + resource.TestCheckResourceAttr(resourceName, "storage_descriptor.0.columns.0.name", "my_column_1"), + resource.TestCheckResourceAttr(resourceName, "storage_descriptor.0.columns.0.parameters.%", "1"), + resource.TestCheckResourceAttr(resourceName, "storage_descriptor.0.columns.0.parameters.param2", "param2_val"), ), }, { @@ -742,6 +772,87 @@ resource "aws_glue_catalog_table" "test" { `, rName) } +func testAccGlueCatalogTableColumnParameters(rName string) string { + return fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_catalog_table" "test" { + name = %[1]q + database_name = aws_glue_catalog_database.test.name + owner = "my_owner" + retention = 1 + table_type = "VIRTUAL_VIEW" + view_expanded_text = "view_expanded_text_1" + view_original_text = "view_original_text_1" + + storage_descriptor { + bucket_columns = ["bucket_column_1"] + compressed = false + input_format = "SequenceFileInputFormat" + location = "my_location" + number_of_buckets = 1 + output_format = "SequenceFileInputFormat" + stored_as_sub_directories = false + + parameters = { + param1 = "param1_val" + } + + columns { + name = "my_column_1" + type = "int" + comment = "my_column1_comment" + + parameters = { + param2 = "param2_val" + } + } + + ser_de_info { + name = "ser_de_name" + + parameters = { + param1 = "param_val_1" + } + + serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" + } + + sort_columns { + column = "my_column_1" + sort_order = 1 + } + + skewed_info { + skewed_column_names = [ + "my_column_1", + ] + + skewed_column_value_location_maps = { + my_column_1 = "my_column_1_val_loc_map" + } + + skewed_column_values = [ + "skewed_val_1", + ] + } + } + + partition_keys { + name = "my_column_1" + type = "int" + comment = "my_column_1_comment" + } + + parameters = { + param1 = "param1_val" + } +} +`, rName) +} + func testAccCheckGlueTableDestroy(s *terraform.State) error { conn := testAccProvider.Meta().(*AWSClient).glueconn diff --git a/website/docs/r/glue_catalog_table.html.markdown b/website/docs/r/glue_catalog_table.html.markdown index af6814b6249..fe801ddc321 100644 --- a/website/docs/r/glue_catalog_table.html.markdown +++ b/website/docs/r/glue_catalog_table.html.markdown @@ -90,14 +90,20 @@ The following arguments are supported: * `description` - (Optional) Description of the table. * `owner` - (Optional) Owner of the table. * `retention` - (Optional) Retention time for this table. -* `storage_descriptor` - (Optional) A [storage descriptor](#storage_descriptor) object containing information about the physical storage of this table. You can refer to the [Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-tables.html#aws-glue-api-catalog-tables-StorageDescriptor) for a full explanation of this object. -* `partition_keys` - (Optional) A list of columns by which the table is partitioned. Only primitive types are supported as partition keys. +* `storage_descriptor` - (Optional) A [storage descriptor](#storage-descriptor) object containing information about the physical storage of this table. You can refer to the [Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-tables.html#aws-glue-api-catalog-tables-StorageDescriptor) for a full explanation of this object. +* `partition_keys` - (Optional) A list of columns by which the table is partitioned. Only primitive types are supported as partition keys. see [Partition Keys](#partition-keys) below. * `view_original_text` - (Optional) If the table is a view, the original text of the view; otherwise null. * `view_expanded_text` - (Optional) If the table is a view, the expanded text of the view; otherwise null. * `table_type` - (Optional) The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc.). While optional, some Athena DDL queries such as `ALTER TABLE` and `SHOW CREATE TABLE` will fail if this argument is empty. * `parameters` - (Optional) Properties associated with this table, as a list of key-value pairs. -##### storage_descriptor +### Partition Keys + +* `name` - (Required) The name of the Partition Key. +* `type` - (Optional) The datatype of data in the Partition Key. +* `comment` - (Optional) Free-form text comment. + +### Storage Descriptor * `columns` - (Optional) A list of the [Columns](#column) in the table. * `location` - (Optional) The physical location of the table. By default this takes the form of the warehouse location, followed by the database location in the warehouse, followed by the table name. @@ -105,31 +111,32 @@ The following arguments are supported: * `output_format` - (Optional) The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format. * `compressed` - (Optional) True if the data in the table is compressed, or False if not. * `number_of_buckets` - (Optional) Must be specified if the table contains any dimension columns. -* `ser_de_info` - (Optional) [Serialization/deserialization (SerDe)](#ser_de_info) information. +* `ser_de_info` - (Optional) [Serialization/deserialization (SerDe)](#ser-de-info) information. * `bucket_columns` - (Optional) A list of reducer grouping columns, clustering columns, and bucketing columns in the table. -* `sort_columns` - (Optional) A list of [Order](#sort_column) objects specifying the sort order of each bucket in the table. +* `sort_columns` - (Optional) A list of [Order](#sort-column) objects specifying the sort order of each bucket in the table. * `parameters` - (Optional) User-supplied properties in key-value form. * `skewed_info` - (Optional) Information about values that appear very frequently in a column (skewed values). * `stored_as_sub_directories` - (Optional) True if the table data is stored in subdirectories, or False if not. -##### column +##### Column * `name` - (Required) The name of the Column. * `type` - (Optional) The datatype of data in the Column. * `comment` - (Optional) Free-form text comment. +* `parameters` - (Optional) These key-value pairs define properties associated with the column. -##### ser_de_info +##### Ser De Info * `name` - (Optional) Name of the SerDe. * `parameters` - (Optional) A map of initialization parameters for the SerDe, in key-value form. * `serialization_library` - (Optional) Usually the class that implements the SerDe. An example is: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe. -##### sort_columns +##### Sort Columns * `column` - (Required) The name of the column. * `sort_order` - (Required) Indicates that the column is sorted in ascending order (== 1), or in descending order (==0). -##### skewed_info +##### Skewed Info * `skewed_column_names` - (Optional) A list of names of columns that contain skewed values. * `skewed_column_value_location_maps` - (Optional) A list of values that appear so frequently as to be considered skewed.