| 1 |
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# https://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
import logging |
| 14 |
|
| 15 |
logger = logging.getLogger(__name__) |
| 16 |
|
| 17 |
|
| 18 |
def register_table_methods(base_classes, **kwargs): |
| 19 |
base_classes.insert(0, TableResource) |
| 20 |
|
| 21 |
|
| 22 |
# This class can be used to add any additional methods we want |
| 23 |
# onto a table resource. Ideally to avoid creating a new |
| 24 |
# base class for every method we can just update this |
| 25 |
# class instead. Just be sure to move the bulk of the |
| 26 |
# actual method implementation to another class. |
| 27 |
class TableResource: |
| 28 |
def __init__(self, *args, **kwargs): |
| 29 |
super().__init__(*args, **kwargs) |
| 30 |
|
| 31 |
def batch_writer(self, overwrite_by_pkeys=None): |
| 32 |
"""Create a batch writer object. |
| 33 |
|
| 34 |
This method creates a context manager for writing |
| 35 |
objects to Amazon DynamoDB in batch. |
| 36 |
|
| 37 |
The batch writer will automatically handle buffering and sending items |
| 38 |
in batches. In addition, the batch writer will also automatically |
| 39 |
handle any unprocessed items and resend them as needed. All you need |
| 40 |
to do is call ``put_item`` for any items you want to add, and |
| 41 |
``delete_item`` for any items you want to delete. |
| 42 |
|
| 43 |
Example usage:: |
| 44 |
|
| 45 |
with table.batch_writer() as batch: |
| 46 |
for _ in range(1000000): |
| 47 |
batch.put_item(Item={'HashKey': '...', |
| 48 |
'Otherstuff': '...'}) |
| 49 |
# You can also delete_items in a batch. |
| 50 |
batch.delete_item(Key={'HashKey': 'SomeHashKey'}) |
| 51 |
|
| 52 |
:type overwrite_by_pkeys: list(string) |
| 53 |
:param overwrite_by_pkeys: De-duplicate request items in buffer |
| 54 |
if match new request item on specified primary keys. i.e |
| 55 |
``["partition_key1", "sort_key2", "sort_key3"]`` |
| 56 |
|
| 57 |
""" |
| 58 |
return BatchWriter( |
| 59 |
self.name, self.meta.client, overwrite_by_pkeys=overwrite_by_pkeys |
| 60 |
) |
| 61 |
|
| 62 |
|
| 63 |
class BatchWriter: |
| 64 |
"""Automatically handle batch writes to DynamoDB for a single table.""" |
| 65 |
|
| 66 |
def __init__( |
| 67 |
self, table_name, client, flush_amount=25, overwrite_by_pkeys=None |
| 68 |
): |
| 69 |
""" |
| 70 |
|
| 71 |
:type table_name: str |
| 72 |
:param table_name: The name of the table. The class handles |
| 73 |
batch writes to a single table. |
| 74 |
|
| 75 |
:type client: ``botocore.client.Client`` |
| 76 |
:param client: A botocore client. Note this client |
| 77 |
**must** have the dynamodb customizations applied |
| 78 |
to it for transforming AttributeValues into the |
| 79 |
wire protocol. What this means in practice is that |
| 80 |
you need to use a client that comes from a DynamoDB |
| 81 |
resource if you're going to instantiate this class |
| 82 |
directly, i.e |
| 83 |
``boto3.resource('dynamodb').Table('foo').meta.client``. |
| 84 |
|
| 85 |
:type flush_amount: int |
| 86 |
:param flush_amount: The number of items to keep in |
| 87 |
a local buffer before sending a batch_write_item |
| 88 |
request to DynamoDB. |
| 89 |
|
| 90 |
:type overwrite_by_pkeys: list(string) |
| 91 |
:param overwrite_by_pkeys: De-duplicate request items in buffer |
| 92 |
if match new request item on specified primary keys. i.e |
| 93 |
``["partition_key1", "sort_key2", "sort_key3"]`` |
| 94 |
|
| 95 |
""" |
| 96 |
self._table_name = table_name |
| 97 |
self._client = client |
| 98 |
self._items_buffer = [] |
| 99 |
self._flush_amount = flush_amount |
| 100 |
self._overwrite_by_pkeys = overwrite_by_pkeys |
| 101 |
|
| 102 |
def put_item(self, Item): |
| 103 |
self._add_request_and_process({'PutRequest': {'Item': Item}}) |
| 104 |
|
| 105 |
def delete_item(self, Key): |
| 106 |
self._add_request_and_process({'DeleteRequest': {'Key': Key}}) |
| 107 |
|
| 108 |
def _add_request_and_process(self, request): |
| 109 |
if self._overwrite_by_pkeys: |
| 110 |
self._remove_dup_pkeys_request_if_any(request) |
| 111 |
self._items_buffer.append(request) |
| 112 |
self._flush_if_needed() |
| 113 |
|
| 114 |
def _remove_dup_pkeys_request_if_any(self, request): |
| 115 |
pkey_values_new = self._extract_pkey_values(request) |
| 116 |
for item in self._items_buffer: |
| 117 |
if self._extract_pkey_values(item) == pkey_values_new: |
| 118 |
self._items_buffer.remove(item) |
| 119 |
logger.debug( |
| 120 |
"With overwrite_by_pkeys enabled, skipping request:%s", |
| 121 |
item, |
| 122 |
) |
| 123 |
|
| 124 |
def _extract_pkey_values(self, request): |
| 125 |
if request.get('PutRequest'): |
| 126 |
return [ |
| 127 |
request['PutRequest']['Item'][key] |
| 128 |
for key in self._overwrite_by_pkeys |
| 129 |
] |
| 130 |
elif request.get('DeleteRequest'): |
| 131 |
return [ |
| 132 |
request['DeleteRequest']['Key'][key] |
| 133 |
for key in self._overwrite_by_pkeys |
| 134 |
] |
| 135 |
return None |
| 136 |
|
| 137 |
def _flush_if_needed(self): |
| 138 |
if len(self._items_buffer) >= self._flush_amount: |
| 139 |
self._flush() |
| 140 |
|
| 141 |
def _flush(self): |
| 142 |
items_to_send = self._items_buffer[: self._flush_amount] |
| 143 |
self._items_buffer = self._items_buffer[self._flush_amount :] |
| 144 |
response = self._client.batch_write_item( |
| 145 |
RequestItems={self._table_name: items_to_send} |
| 146 |
) |
| 147 |
unprocessed_items = response['UnprocessedItems'] |
| 148 |
if not unprocessed_items: |
| 149 |
unprocessed_items = {} |
| 150 |
item_list = unprocessed_items.get(self._table_name, []) |
| 151 |
# Any unprocessed_items are immediately added to the |
| 152 |
# next batch we send. |
| 153 |
self._items_buffer.extend(item_list) |
| 154 |
logger.debug( |
| 155 |
"Batch write sent %s, unprocessed: %s", |
| 156 |
len(items_to_send), |
| 157 |
len(self._items_buffer), |
| 158 |
) |
| 159 |
|
| 160 |
def __enter__(self): |
| 161 |
return self |
| 162 |
|
| 163 |
def __exit__(self, exc_type, exc_value, tb): |
| 164 |
# When we exit, we need to keep flushing whatever's left |
| 165 |
# until there's nothing left in our items buffer. |
| 166 |
while self._items_buffer: |
| 167 |
self._flush() |