From fff1a7504afb50d79fcdefc10306f29c4cc3db4a Mon Sep 17 00:00:00 2001 From: "David L. Miller" <45697098+dlm6693@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:46:58 -0400 Subject: [PATCH 1/3] dynamodb BatchWriter._flush bug (#3279) * dynamodb BatchWriter._flush bug * updated test comments * check if unprocessed items is None * pr feedback * pr feedback * updated tests --- boto3/dynamodb/table.py | 13 ++-- tests/unit/dynamodb/test_table.py | 108 ++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/boto3/dynamodb/table.py b/boto3/dynamodb/table.py index d96ec8f5ee..931296bc09 100644 --- a/boto3/dynamodb/table.py +++ b/boto3/dynamodb/table.py @@ -145,13 +145,12 @@ def _flush(self): RequestItems={self._table_name: items_to_send} ) unprocessed_items = response['UnprocessedItems'] - - if unprocessed_items and unprocessed_items[self._table_name]: - # Any unprocessed_items are immediately added to the - # next batch we send. - self._items_buffer.extend(unprocessed_items[self._table_name]) - else: - self._items_buffer = [] + if not unprocessed_items: + unprocessed_items = {} + item_list = unprocessed_items.get(self._table_name, []) + # Any unprocessed_items are immediately added to the + # next batch we send. + self._items_buffer.extend(item_list) logger.debug( "Batch write sent %s, unprocessed: %s", len(items_to_send), diff --git a/tests/unit/dynamodb/test_table.py b/tests/unit/dynamodb/test_table.py index 56f8dfcde7..f78da6f4e2 100644 --- a/tests/unit/dynamodb/test_table.py +++ b/tests/unit/dynamodb/test_table.py @@ -411,3 +411,111 @@ def test_auto_dedup_for_dup_requests(self): } } self.assert_batch_write_calls_are([first_batch, second_batch]) + + def test_added_unsent_request_not_flushed_put(self): + # If n requests that get sent fail to process where n = flush_amount + # and at least one more request gets created before the second attempt, + # then previously if n requests were successful on the next run and + # returned an empty dict, _item_buffer would be emptied before sending + # the next batch of n requests + self.client.batch_write_item.side_effect = [ + { + 'UnprocessedItems': { + self.table_name: [ + {'PutRequest': {'Item': {'Hash': 'foo1'}}}, + {'PutRequest': {'Item': {'Hash': 'foo2'}}}, + ], + }, + }, + { + 'UnprocessedItems': {}, + }, + { + 'UnprocessedItems': {}, + }, + ] + self.batch_writer.put_item({'Hash': 'foo1'}) + self.batch_writer.put_item({'Hash': 'foo2'}) + self.batch_writer.put_item({'Hash': 'foo3'}) + self.assertIn( + {'PutRequest': {'Item': {'Hash': 'foo3'}}}, + self.batch_writer._items_buffer, + ) + batch = { + 'RequestItems': { + self.table_name: [ + {'PutRequest': {'Item': {'Hash': 'foo1'}}}, + {'PutRequest': {'Item': {'Hash': 'foo2'}}}, + ] + } + } + final_batch = { + 'RequestItems': { + self.table_name: [ + {'PutRequest': {'Item': {'Hash': 'foo3'}}}, + {'PutRequest': {'Item': {'Hash': 'foo4'}}}, + ] + } + } + # same batch sent twice since all failed on first try + # and flush_items = 2 + self.assert_batch_write_calls_are([batch, batch]) + # test that the next two items get sent + self.batch_writer.put_item({'Hash': 'foo4'}) + self.assert_batch_write_calls_are([batch, batch, final_batch]) + # the buffer should be empty now + self.assertEqual(self.batch_writer._items_buffer, []) + + def test_added_unsent_request_not_flushed_delete(self): + # If n requests that get sent fail to process where n = flush_amount + # and at least one more request gets created before the second attempt, + # then previously if n requests were successful on the next run and + # returned an empty dict, _item_buffer would be emptied before sending + # the next batch of n requests + self.client.batch_write_item.side_effect = [ + { + 'UnprocessedItems': { + self.table_name: [ + {'DeleteRequest': {'Key': {'Hash': 'foo1'}}}, + {'DeleteRequest': {'Key': {'Hash': 'foo2'}}}, + ], + }, + }, + { + 'UnprocessedItems': {}, + }, + { + 'UnprocessedItems': {}, + }, + ] + self.batch_writer.delete_item({'Hash': 'foo1'}) + self.batch_writer.delete_item({'Hash': 'foo2'}) + self.batch_writer.delete_item({'Hash': 'foo3'}) + self.assertIn( + {'DeleteRequest': {'Key': {'Hash': 'foo3'}}}, + self.batch_writer._items_buffer, + ) + batch = { + 'RequestItems': { + self.table_name: [ + {'DeleteRequest': {'Key': {'Hash': 'foo1'}}}, + {'DeleteRequest': {'Key': {'Hash': 'foo2'}}}, + ] + } + } + final_batch = { + 'RequestItems': { + self.table_name: [ + {'DeleteRequest': {'Key': {'Hash': 'foo3'}}}, + {'DeleteRequest': {'Key': {'Hash': 'foo4'}}}, + ] + } + } + # same batch sent twice since all failed on first try + # and flush_items = 2 + self.assert_batch_write_calls_are([batch, batch]) + # test that the next two items get sent + self.batch_writer.delete_item({'Hash': 'foo4'}) + self.assert_batch_write_calls_are([batch, batch, final_batch]) + # the buffer should be empty now + self.assertEqual(self.batch_writer._items_buffer, []) From 5c9e7dceb8a2671a4830044e50ca667da89e1fc0 Mon Sep 17 00:00:00 2001 From: aws-sdk-python-automation Date: Mon, 6 Jun 2022 18:25:29 +0000 Subject: [PATCH 2/3] Add changelog entries from botocore --- .../next-release/api-change-chimesdkmessaging-34807.json | 5 +++++ .changes/next-release/api-change-connect-77805.json | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changes/next-release/api-change-chimesdkmessaging-34807.json create mode 100644 .changes/next-release/api-change-connect-77805.json diff --git a/.changes/next-release/api-change-chimesdkmessaging-34807.json b/.changes/next-release/api-change-chimesdkmessaging-34807.json new file mode 100644 index 0000000000..07a3c17ecf --- /dev/null +++ b/.changes/next-release/api-change-chimesdkmessaging-34807.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``chime-sdk-messaging``", + "description": "[``botocore``] This release adds support for searching channels by members via the SearchChannels API, removes required restrictions for Name and Mode in UpdateChannel API and enhances CreateChannel API by exposing member and moderator list as well as channel id as optional parameters." +} diff --git a/.changes/next-release/api-change-connect-77805.json b/.changes/next-release/api-change-connect-77805.json new file mode 100644 index 0000000000..256f59f165 --- /dev/null +++ b/.changes/next-release/api-change-connect-77805.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``connect``", + "description": "[``botocore``] This release adds a new API, GetCurrentUserData, which returns real-time details about users' current activity." +} From 92a1eb6baaf095fc94c9fdf9e438efa8cc01147a Mon Sep 17 00:00:00 2001 From: aws-sdk-python-automation Date: Mon, 6 Jun 2022 18:25:48 +0000 Subject: [PATCH 3/3] Bumping version to 1.24.3 --- .changes/1.24.3.json | 12 ++++++++++++ .../api-change-chimesdkmessaging-34807.json | 5 ----- .changes/next-release/api-change-connect-77805.json | 5 ----- CHANGELOG.rst | 7 +++++++ boto3/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 .changes/1.24.3.json delete mode 100644 .changes/next-release/api-change-chimesdkmessaging-34807.json delete mode 100644 .changes/next-release/api-change-connect-77805.json diff --git a/.changes/1.24.3.json b/.changes/1.24.3.json new file mode 100644 index 0000000000..452445dac9 --- /dev/null +++ b/.changes/1.24.3.json @@ -0,0 +1,12 @@ +[ + { + "category": "``chime-sdk-messaging``", + "description": "[``botocore``] This release adds support for searching channels by members via the SearchChannels API, removes required restrictions for Name and Mode in UpdateChannel API and enhances CreateChannel API by exposing member and moderator list as well as channel id as optional parameters.", + "type": "api-change" + }, + { + "category": "``connect``", + "description": "[``botocore``] This release adds a new API, GetCurrentUserData, which returns real-time details about users' current activity.", + "type": "api-change" + } +] \ No newline at end of file diff --git a/.changes/next-release/api-change-chimesdkmessaging-34807.json b/.changes/next-release/api-change-chimesdkmessaging-34807.json deleted file mode 100644 index 07a3c17ecf..0000000000 --- a/.changes/next-release/api-change-chimesdkmessaging-34807.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``chime-sdk-messaging``", - "description": "[``botocore``] This release adds support for searching channels by members via the SearchChannels API, removes required restrictions for Name and Mode in UpdateChannel API and enhances CreateChannel API by exposing member and moderator list as well as channel id as optional parameters." -} diff --git a/.changes/next-release/api-change-connect-77805.json b/.changes/next-release/api-change-connect-77805.json deleted file mode 100644 index 256f59f165..0000000000 --- a/.changes/next-release/api-change-connect-77805.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``connect``", - "description": "[``botocore``] This release adds a new API, GetCurrentUserData, which returns real-time details about users' current activity." -} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d0474b1a9..31d2389bbc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ CHANGELOG ========= +1.24.3 +====== + +* api-change:``chime-sdk-messaging``: [``botocore``] This release adds support for searching channels by members via the SearchChannels API, removes required restrictions for Name and Mode in UpdateChannel API and enhances CreateChannel API by exposing member and moderator list as well as channel id as optional parameters. +* api-change:``connect``: [``botocore``] This release adds a new API, GetCurrentUserData, which returns real-time details about users' current activity. + + 1.24.2 ====== diff --git a/boto3/__init__.py b/boto3/__init__.py index 4f129b8f51..f81d227b49 100644 --- a/boto3/__init__.py +++ b/boto3/__init__.py @@ -17,7 +17,7 @@ from boto3.session import Session __author__ = 'Amazon Web Services' -__version__ = '1.24.2' +__version__ = '1.24.3' # The default Boto3 session; autoloaded when needed. diff --git a/setup.cfg b/setup.cfg index 3b93445a51..6ac8147dc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ universal = 0 [metadata] requires_dist = - botocore>=1.27.2,<1.28.0 + botocore>=1.27.3,<1.28.0 jmespath>=0.7.1,<2.0.0 s3transfer>=0.6.0,<0.7.0 diff --git a/setup.py b/setup.py index d596cc0cf0..5a26d86de5 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ requires = [ - 'botocore>=1.27.2,<1.28.0', + 'botocore>=1.27.3,<1.28.0', 'jmespath>=0.7.1,<2.0.0', 's3transfer>=0.6.0,<0.7.0', ]