From 80c4609c4a58fee1fe591f88ba4e1d888202c5dd Mon Sep 17 00:00:00 2001 From: Nils Persson Date: Wed, 20 Sep 2023 22:46:30 -0400 Subject: [PATCH] fix: allow full replacement of bq schema with table_schema when if_exists="replace" (#670) A previous issue (#315) noted that when using `if_exists="append"`, pandas should retrieve the existing schema from BQ and update the user-supplied `table_schema` to ensure that all column modes (REQUIRED/NULLABLE) match those of the existing table. The commit that fixed this issue applies this policy to *all* write_dispositions, but it is only applicable when appending. When replacing, the user should be able to overwrite with new datatypes. --- pandas_gbq/gbq.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index edf82734..2e3a6f86 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -1205,12 +1205,14 @@ def to_gbq( ) table_connector.create(table_id, table_schema) else: - # Convert original schema (the schema that already exists) to pandas-gbq API format - original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) - - # Update the local `table_schema` so mode (NULLABLE/REQUIRED) - # matches. See: https://github.com/pydata/pandas-gbq/issues/315 - table_schema = pandas_gbq.schema.update_schema(table_schema, original_schema) + if write_disposition == "WRITE_APPEND": + # If appending, ensure types and modes of existing columns match supplied schema + # Convert original schema (the schema that already exists) to pandas-gbq API format + original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) + + # Update the local `table_schema` so mode (NULLABLE/REQUIRED) + # matches. See: https://github.com/pydata/pandas-gbq/issues/315 + table_schema = pandas_gbq.schema.update_schema(table_schema, original_schema) if dataframe.empty: # Create the table (if needed), but don't try to run a load job with an