Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

r/glue_catalog_table - add parameters argument to columns + validations #16052

Merged
merged 7 commits into from
Nov 17, 2020
Merged

r/glue_catalog_table - add parameters argument to columns + validations #16052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a9b2735
validations
DrFaust92 Nov 4, 2020
b7d29ac
use %w for errors + add parameteres
DrFaust92 Nov 5, 2020
d578b91
add catalog id check
DrFaust92 Nov 5, 2020
5d2e942
test
DrFaust92 Nov 5, 2020
478b15f
improve docs
DrFaust92 Nov 5, 2020
77a8848
fmt
DrFaust92 Nov 5, 2020
be4ef49
Apply suggestions from code review
DrFaust92 Nov 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 96 additions & 88 deletions aws/resource_aws_glue_catalog_table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"fmt"
"log"
"regexp"
"strings"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -39,13 +40,18 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Required: true,
},
"description": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 2048),
},
"name": {
Type: schema.TypeString,
ForceNew: true,
Required: true,
ValidateFunc: validation.All(
validation.StringLenBetween(1, 255),
validation.StringDoesNotMatch(regexp.MustCompile(`[A-Z]`), "uppercase characters cannot be used"),
),
},
"owner": {
Type: schema.TypeString,
Expand All @@ -62,23 +68,27 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"comment": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 255),
},
"name": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"type": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 131072),
},
},
},
},
"retention": {
Type: schema.TypeInt,
Optional: true,
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntAtLeast(0),
},
"storage_descriptor": {
Type: schema.TypeList,
Expand All @@ -89,24 +99,35 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"bucket_columns": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
Elem: &schema.Schema{
Type: schema.TypeString,
ValidateFunc: validation.StringLenBetween(1, 255),
},
},
"columns": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"comment": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 255),
},
"name": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"type": {
Type: schema.TypeString,
"parameters": {
Type: schema.TypeMap,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"type": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 131072),
},
},
},
Expand Down Expand Up @@ -145,7 +166,7 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"name": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"parameters": {
Type: schema.TypeMap,
Expand All @@ -169,7 +190,10 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"skewed_column_names": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
Elem: &schema.Schema{
Type: schema.TypeString,
ValidateFunc: validation.StringLenBetween(1, 255),
},
},
"skewed_column_values": {
Type: schema.TypeList,
Expand All @@ -190,12 +214,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"column": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"sort_order": {
Type: schema.TypeInt,
Required: true,
Type: schema.TypeInt,
Required: true,
ValidateFunc: validation.IntInSlice([]int{0, 1}),
},
},
},
Expand All @@ -212,12 +238,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Optional: true,
},
"view_original_text": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 409600),
},
"view_expanded_text": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 409600),
},
},
}
Expand All @@ -243,9 +271,10 @@ func resourceAwsGlueCatalogTableCreate(d *schema.ResourceData, meta interface{})
TableInput: expandGlueTableInput(d),
}

log.Printf("[DEBUG] Glue catalog table input: %#v", input)
_, err := conn.CreateTable(input)
if err != nil {
return fmt.Errorf("Error creating Catalog Table: %s", err)
return fmt.Errorf("Error creating Glue Catalog Table: %w", err)
}

d.SetId(fmt.Sprintf("%s:%s:%s", catalogID, dbName, name))
Expand Down Expand Up @@ -276,39 +305,40 @@ func resourceAwsGlueCatalogTableRead(d *schema.ResourceData, meta interface{}) e
return nil
}

return fmt.Errorf("Error reading Glue Catalog Table: %s", err)
return fmt.Errorf("Error reading Glue Catalog Table: %w", err)
}

table := out.Table
tableArn := arn.ARN{
Partition: meta.(*AWSClient).partition,
Service: "glue",
Region: meta.(*AWSClient).region,
AccountID: meta.(*AWSClient).accountid,
Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(out.Table.Name)),
Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(table.Name)),
}.String()
d.Set("arn", tableArn)

d.Set("name", out.Table.Name)
d.Set("name", table.Name)
d.Set("catalog_id", catalogID)
d.Set("database_name", dbName)
d.Set("description", out.Table.Description)
d.Set("owner", out.Table.Owner)
d.Set("retention", out.Table.Retention)
d.Set("description", table.Description)
d.Set("owner", table.Owner)
d.Set("retention", table.Retention)

if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(out.Table.StorageDescriptor)); err != nil {
return fmt.Errorf("error setting storage_descriptor: %s", err)
if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(table.StorageDescriptor)); err != nil {
return fmt.Errorf("error setting storage_descriptor: %w", err)
}

if err := d.Set("partition_keys", flattenGlueColumns(out.Table.PartitionKeys)); err != nil {
return fmt.Errorf("error setting partition_keys: %s", err)
if err := d.Set("partition_keys", flattenGlueColumns(table.PartitionKeys)); err != nil {
return fmt.Errorf("error setting partition_keys: %w", err)
}

d.Set("view_original_text", out.Table.ViewOriginalText)
d.Set("view_expanded_text", out.Table.ViewExpandedText)
d.Set("table_type", out.Table.TableType)
d.Set("view_original_text", table.ViewOriginalText)
d.Set("view_expanded_text", table.ViewExpandedText)
d.Set("table_type", table.TableType)

if err := d.Set("parameters", aws.StringValueMap(out.Table.Parameters)); err != nil {
return fmt.Errorf("error setting parameters: %s", err)
if err := d.Set("parameters", aws.StringValueMap(table.Parameters)); err != nil {
return fmt.Errorf("error setting parameters: %w", err)
}

return nil
Expand All @@ -329,7 +359,7 @@ func resourceAwsGlueCatalogTableUpdate(d *schema.ResourceData, meta interface{})
}

if _, err := conn.UpdateTable(updateTableInput); err != nil {
return fmt.Errorf("Error updating Glue Catalog Table: %s", err)
return fmt.Errorf("Error updating Glue Catalog Table: %w", err)
}

return resourceAwsGlueCatalogTableRead(d, meta)
Expand All @@ -350,7 +380,7 @@ func resourceAwsGlueCatalogTableDelete(d *schema.ResourceData, meta interface{})
DatabaseName: aws.String(dbName),
})
if err != nil {
return fmt.Errorf("Error deleting Glue Catalog Table: %s", err.Error())
return fmt.Errorf("Error deleting Glue Catalog Table: %w", err)
}
return nil
}
Expand All @@ -377,8 +407,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput {
}

if v, ok := d.GetOk("partition_keys"); ok {
columns := expandGlueColumns(v.([]interface{}))
tableInput.PartitionKeys = columns
tableInput.PartitionKeys = expandGlueColumns(v.([]interface{}))
}

if v, ok := d.GetOk("view_original_text"); ok {
Expand All @@ -394,11 +423,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput {
}

if v, ok := d.GetOk("parameters"); ok {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
tableInput.Parameters = aws.StringMap(paramsMap)
tableInput.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

return tableInput
Expand All @@ -413,8 +438,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
storageDescriptor := &glue.StorageDescriptor{}

if v, ok := s["columns"]; ok {
columns := expandGlueColumns(v.([]interface{}))
storageDescriptor.Columns = columns
storageDescriptor.Columns = expandGlueColumns(v.([]interface{}))
}

if v, ok := s["location"]; ok {
Expand Down Expand Up @@ -442,11 +466,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
}

if v, ok := s["bucket_columns"]; ok {
bucketColumns := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
bucketColumns[i] = fmt.Sprint(item)
}
storageDescriptor.BucketColumns = aws.StringSlice(bucketColumns)
storageDescriptor.BucketColumns = expandStringList(v.([]interface{}))
}

if v, ok := s["sort_columns"]; ok {
Expand All @@ -458,11 +478,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
}

if v, ok := s["parameters"]; ok {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
storageDescriptor.Parameters = aws.StringMap(paramsMap)
storageDescriptor.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

if v, ok := s["stored_as_sub_directories"]; ok {
Expand All @@ -489,6 +505,10 @@ func expandGlueColumns(columns []interface{}) []*glue.Column {
column.Type = aws.String(v.(string))
}

if v, ok := elementMap["parameters"]; ok {
column.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

columnSlice = append(columnSlice, column)
}

Expand All @@ -508,11 +528,7 @@ func expandGlueSerDeInfo(l []interface{}) *glue.SerDeInfo {
}

if v := s["parameters"]; len(v.(map[string]interface{})) > 0 {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
serDeInfo.Parameters = aws.StringMap(paramsMap)
serDeInfo.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

if v := s["serialization_library"]; len(v.(string)) > 0 {
Expand Down Expand Up @@ -551,27 +567,15 @@ func expandGlueSkewedInfo(l []interface{}) *glue.SkewedInfo {
skewedInfo := &glue.SkewedInfo{}

if v, ok := s["skewed_column_names"]; ok {
columnsSlice := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
columnsSlice[i] = fmt.Sprint(item)
}
skewedInfo.SkewedColumnNames = aws.StringSlice(columnsSlice)
skewedInfo.SkewedColumnNames = expandStringList(v.([]interface{}))
}

if v, ok := s["skewed_column_value_location_maps"]; ok {
typeMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
typeMap[key] = value.(string)
}
skewedInfo.SkewedColumnValueLocationMaps = aws.StringMap(typeMap)
skewedInfo.SkewedColumnValueLocationMaps = stringMapToPointers(v.(map[string]interface{}))
}

if v, ok := s["skewed_column_values"]; ok {
columnsSlice := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
columnsSlice[i] = fmt.Sprint(item)
}
skewedInfo.SkewedColumnValues = aws.StringSlice(columnsSlice)
skewedInfo.SkewedColumnValues = expandStringList(v.([]interface{}))
}

return skewedInfo
Expand Down Expand Up @@ -605,8 +609,8 @@ func flattenGlueStorageDescriptor(s *glue.StorageDescriptor) []map[string]interf
return storageDescriptors
}

func flattenGlueColumns(cs []*glue.Column) []map[string]string {
columnsSlice := make([]map[string]string, len(cs))
func flattenGlueColumns(cs []*glue.Column) []map[string]interface{} {
columnsSlice := make([]map[string]interface{}, len(cs))
if len(cs) > 0 {
for i, v := range cs {
columnsSlice[i] = flattenGlueColumn(v)
Expand All @@ -616,8 +620,8 @@ func flattenGlueColumns(cs []*glue.Column) []map[string]string {
return columnsSlice
}

func flattenGlueColumn(c *glue.Column) map[string]string {
column := make(map[string]string)
func flattenGlueColumn(c *glue.Column) map[string]interface{} {
column := make(map[string]interface{})

if c == nil {
return column
Expand All @@ -635,6 +639,10 @@ func flattenGlueColumn(c *glue.Column) map[string]string {
column["comment"] = v
}

if v := c.Parameters; v != nil {
column["parameters"] = aws.StringValueMap(v)
}

return column
}

Expand Down
Loading