From 535017f2ce3287ffe58f09e8c4196127aaed1015 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Wed, 2 Aug 2017 12:02:40 +0900 Subject: [PATCH 1/6] Add timeout support --- pandas_gbq/gbq.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 452de5ef..d362356b 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -172,6 +172,13 @@ class NotFoundException(ValueError): pass +class QueryTimeout(ValueError): + """ + Raised when the query job timeout + """ + pass + + class StreamingInsertError(ValueError): """ Raised when BigQuery reports a streaming insert error. @@ -543,6 +550,10 @@ def run_query(self, query, **kwargs): except HttpError as ex: self.process_http_error(ex) + timeoutMs = job_config['query'].get('timeoutMs') + if timeoutMs and timeoutMs < self.get_elapsed_seconds()*1000: + raise QueryTimeout('Query timeout: {} ms'.format(timeoutMs)) + if self.verbose: if query_reply['cacheHit']: self._print('Query done.\nCache hit.\n') From 289d2aad3e78bae144f027bdc104db7a4b19ffa1 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Wed, 2 Aug 2017 13:17:03 +0900 Subject: [PATCH 2/6] Fix a pep8 issue --- pandas_gbq/gbq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index d362356b..92fc7bf8 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -551,7 +551,7 @@ def run_query(self, query, **kwargs): self.process_http_error(ex) timeoutMs = job_config['query'].get('timeoutMs') - if timeoutMs and timeoutMs < self.get_elapsed_seconds()*1000: + if timeoutMs and timeoutMs < self.get_elapsed_seconds() * 1000: raise QueryTimeout('Query timeout: {} ms'.format(timeoutMs)) if self.verbose: From 686ba6740bbe315a906eddd382c68a639b722565 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Wed, 2 Aug 2017 16:19:45 +0900 Subject: [PATCH 3/6] Check before fetch results --- pandas_gbq/gbq.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 92fc7bf8..cb9ac403 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -543,6 +543,11 @@ def run_query(self, query, **kwargs): while not query_reply.get('jobComplete', False): self.print_elapsed_seconds(' Elapsed', 's. Waiting...') + + timeoutMs = job_config['query'].get('timeoutMs') + if timeoutMs and timeoutMs < self.get_elapsed_seconds() * 1000: + raise QueryTimeout('Query timeout: {} ms'.format(timeoutMs)) + try: query_reply = job_collection.getQueryResults( projectId=job_reference['projectId'], @@ -550,10 +555,6 @@ def run_query(self, query, **kwargs): except HttpError as ex: self.process_http_error(ex) - timeoutMs = job_config['query'].get('timeoutMs') - if timeoutMs and timeoutMs < self.get_elapsed_seconds() * 1000: - raise QueryTimeout('Query timeout: {} ms'.format(timeoutMs)) - if self.verbose: if query_reply['cacheHit']: self._print('Query done.\nCache hit.\n') From f7ba57c6dbabf87d9b28f4991782607b7f05ed53 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Wed, 2 Aug 2017 16:25:01 +0900 Subject: [PATCH 4/6] Add test --- pandas_gbq/tests/test_gbq.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index e8eda1d3..7b95563b 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -884,6 +884,19 @@ def test_configuration_raises_value_error_with_multiple_config(self): private_key=_get_private_key_path(), configuration=config) + def test_timeout_configuration(self): + sql_statement = 'SELECT 1' + config = { + 'query': { + "timeoutMs": 1 + } + } + # Test that QueryTimeout error raises + with pytest.raises(gbq.QueryTimeout): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" assert self.gbq_connector.sizeof_fmt(1024) == "1.0 KB" From 37d1afb303dba18a0fd2566de91c3a9948593ea7 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Fri, 4 Aug 2017 10:44:53 +0900 Subject: [PATCH 5/6] timeoutMs -> timeout_ms --- pandas_gbq/gbq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index cb9ac403..9d1e5cac 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -544,9 +544,9 @@ def run_query(self, query, **kwargs): while not query_reply.get('jobComplete', False): self.print_elapsed_seconds(' Elapsed', 's. Waiting...') - timeoutMs = job_config['query'].get('timeoutMs') - if timeoutMs and timeoutMs < self.get_elapsed_seconds() * 1000: - raise QueryTimeout('Query timeout: {} ms'.format(timeoutMs)) + timeout_ms = job_config['query'].get('timeoutMs') + if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: + raise QueryTimeout('Query timeout: {} ms'.format(timeout_ms)) try: query_reply = job_collection.getQueryResults( From 8622cc84030f52a8eb27bb2f4041f654271218e3 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Fri, 4 Aug 2017 11:06:38 +0900 Subject: [PATCH 6/6] Add changelog --- docs/source/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d71f6287..48ed0b51 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +0.2.1 / ? +------------------ + +- :func:`read_gbq` now handles query configuration `query.timeoutMs` and stop waiting. (:issue:`76`) + 0.2.0 / 2017-07-24 ------------------