| 1 |
from math import ceil |
| 2 |
from operator import attrgetter |
| 3 |
|
| 4 |
from django.core.exceptions import FieldDoesNotExist |
| 5 |
from django.db import ( |
| 6 |
IntegrityError, |
| 7 |
NotSupportedError, |
| 8 |
OperationalError, |
| 9 |
ProgrammingError, |
| 10 |
connection, |
| 11 |
) |
| 12 |
from django.db.models import FileField, Value |
| 13 |
from django.db.models.functions import Lower, Now |
| 14 |
from django.test import ( |
| 15 |
TestCase, |
| 16 |
override_settings, |
| 17 |
skipIfDBFeature, |
| 18 |
skipUnlessDBFeature, |
| 19 |
) |
| 20 |
|
| 21 |
from .models import ( |
| 22 |
BigAutoFieldModel, |
| 23 |
Country, |
| 24 |
FieldsWithDbColumns, |
| 25 |
NoFields, |
| 26 |
NullableFields, |
| 27 |
Pizzeria, |
| 28 |
ProxyCountry, |
| 29 |
ProxyMultiCountry, |
| 30 |
ProxyMultiProxyCountry, |
| 31 |
ProxyProxyCountry, |
| 32 |
RelatedModel, |
| 33 |
Restaurant, |
| 34 |
SmallAutoFieldModel, |
| 35 |
State, |
| 36 |
TwoFields, |
| 37 |
UpsertConflict, |
| 38 |
) |
| 39 |
|
| 40 |
|
| 41 |
class BulkCreateTests(TestCase): |
| 42 |
def setUp(self): |
| 43 |
self.data = [ |
| 44 |
Country(name="United States of America", iso_two_letter="US"), |
| 45 |
Country(name="The Netherlands", iso_two_letter="NL"), |
| 46 |
Country(name="Germany", iso_two_letter="DE"), |
| 47 |
Country(name="Czech Republic", iso_two_letter="CZ"), |
| 48 |
] |
| 49 |
|
| 50 |
def test_simple(self): |
| 51 |
created = Country.objects.bulk_create(self.data) |
| 52 |
self.assertEqual(created, self.data) |
| 53 |
self.assertQuerySetEqual( |
| 54 |
Country.objects.order_by("-name"), |
| 55 |
[ |
| 56 |
"United States of America", |
| 57 |
"The Netherlands", |
| 58 |
"Germany", |
| 59 |
"Czech Republic", |
| 60 |
], |
| 61 |
attrgetter("name"), |
| 62 |
) |
| 63 |
|
| 64 |
created = Country.objects.bulk_create([]) |
| 65 |
self.assertEqual(created, []) |
| 66 |
self.assertEqual(Country.objects.count(), 4) |
| 67 |
|
| 68 |
@skipUnlessDBFeature("has_bulk_insert") |
| 69 |
def test_efficiency(self): |
| 70 |
with self.assertNumQueries(1): |
| 71 |
Country.objects.bulk_create(self.data) |
| 72 |
|
| 73 |
@skipUnlessDBFeature("has_bulk_insert") |
| 74 |
def test_long_non_ascii_text(self): |
| 75 |
""" |
| 76 |
Inserting non-ASCII values with a length in the range 2001 to 4000 |
| 77 |
characters, i.e. 4002 to 8000 bytes, must be set as a CLOB on Oracle |
| 78 |
(#22144). |
| 79 |
""" |
| 80 |
Country.objects.bulk_create([Country(description="Ж" * 3000)]) |
| 81 |
self.assertEqual(Country.objects.count(), 1) |
| 82 |
|
| 83 |
@skipUnlessDBFeature("has_bulk_insert") |
| 84 |
def test_long_and_short_text(self): |
| 85 |
Country.objects.bulk_create( |
| 86 |
[ |
| 87 |
Country(description="a" * 4001, iso_two_letter="A"), |
| 88 |
Country(description="a", iso_two_letter="B"), |
| 89 |
Country(description="Ж" * 2001, iso_two_letter="C"), |
| 90 |
Country(description="Ж", iso_two_letter="D"), |
| 91 |
] |
| 92 |
) |
| 93 |
self.assertEqual(Country.objects.count(), 4) |
| 94 |
|
| 95 |
def test_multi_table_inheritance_unsupported(self): |
| 96 |
expected_message = "Can't bulk create a multi-table inherited model" |
| 97 |
with self.assertRaisesMessage(ValueError, expected_message): |
| 98 |
Pizzeria.objects.bulk_create( |
| 99 |
[ |
| 100 |
Pizzeria(name="The Art of Pizza"), |
| 101 |
] |
| 102 |
) |
| 103 |
with self.assertRaisesMessage(ValueError, expected_message): |
| 104 |
ProxyMultiCountry.objects.bulk_create( |
| 105 |
[ |
| 106 |
ProxyMultiCountry(name="Fillory", iso_two_letter="FL"), |
| 107 |
] |
| 108 |
) |
| 109 |
with self.assertRaisesMessage(ValueError, expected_message): |
| 110 |
ProxyMultiProxyCountry.objects.bulk_create( |
| 111 |
[ |
| 112 |
ProxyMultiProxyCountry(name="Fillory", iso_two_letter="FL"), |
| 113 |
] |
| 114 |
) |
| 115 |
|
| 116 |
def test_proxy_inheritance_supported(self): |
| 117 |
ProxyCountry.objects.bulk_create( |
| 118 |
[ |
| 119 |
ProxyCountry(name="Qwghlm", iso_two_letter="QW"), |
| 120 |
Country(name="Tortall", iso_two_letter="TA"), |
| 121 |
] |
| 122 |
) |
| 123 |
self.assertQuerySetEqual( |
| 124 |
ProxyCountry.objects.all(), |
| 125 |
{"Qwghlm", "Tortall"}, |
| 126 |
attrgetter("name"), |
| 127 |
ordered=False, |
| 128 |
) |
| 129 |
|
| 130 |
ProxyProxyCountry.objects.bulk_create( |
| 131 |
[ |
| 132 |
ProxyProxyCountry(name="Netherlands", iso_two_letter="NT"), |
| 133 |
] |
| 134 |
) |
| 135 |
self.assertQuerySetEqual( |
| 136 |
ProxyProxyCountry.objects.all(), |
| 137 |
{ |
| 138 |
"Qwghlm", |
| 139 |
"Tortall", |
| 140 |
"Netherlands", |
| 141 |
}, |
| 142 |
attrgetter("name"), |
| 143 |
ordered=False, |
| 144 |
) |
| 145 |
|
| 146 |
def test_non_auto_increment_pk(self): |
| 147 |
State.objects.bulk_create( |
| 148 |
[State(two_letter_code=s) for s in ["IL", "NY", "CA", "ME"]] |
| 149 |
) |
| 150 |
self.assertQuerySetEqual( |
| 151 |
State.objects.order_by("two_letter_code"), |
| 152 |
[ |
| 153 |
"CA", |
| 154 |
"IL", |
| 155 |
"ME", |
| 156 |
"NY", |
| 157 |
], |
| 158 |
attrgetter("two_letter_code"), |
| 159 |
) |
| 160 |
|
| 161 |
@skipUnlessDBFeature("has_bulk_insert") |
| 162 |
def test_non_auto_increment_pk_efficiency(self): |
| 163 |
with self.assertNumQueries(1): |
| 164 |
State.objects.bulk_create( |
| 165 |
[State(two_letter_code=s) for s in ["IL", "NY", "CA", "ME"]] |
| 166 |
) |
| 167 |
self.assertQuerySetEqual( |
| 168 |
State.objects.order_by("two_letter_code"), |
| 169 |
[ |
| 170 |
"CA", |
| 171 |
"IL", |
| 172 |
"ME", |
| 173 |
"NY", |
| 174 |
], |
| 175 |
attrgetter("two_letter_code"), |
| 176 |
) |
| 177 |
|
| 178 |
@skipIfDBFeature("allows_auto_pk_0") |
| 179 |
def test_zero_as_autoval(self): |
| 180 |
""" |
| 181 |
Zero as id for AutoField should raise exception in MySQL, because MySQL |
| 182 |
does not allow zero for automatic primary key if the |
| 183 |
NO_AUTO_VALUE_ON_ZERO SQL mode is not enabled. |
| 184 |
""" |
| 185 |
valid_country = Country(name="Germany", iso_two_letter="DE") |
| 186 |
invalid_country = Country(id=0, name="Poland", iso_two_letter="PL") |
| 187 |
msg = "The database backend does not accept 0 as a value for AutoField." |
| 188 |
with self.assertRaisesMessage(ValueError, msg): |
| 189 |
Country.objects.bulk_create([valid_country, invalid_country]) |
| 190 |
|
| 191 |
def test_batch_same_vals(self): |
| 192 |
# SQLite had a problem where all the same-valued models were |
| 193 |
# collapsed to one insert. |
| 194 |
Restaurant.objects.bulk_create([Restaurant(name="foo") for i in range(0, 2)]) |
| 195 |
self.assertEqual(Restaurant.objects.count(), 2) |
| 196 |
|
| 197 |
def test_large_batch(self): |
| 198 |
TwoFields.objects.bulk_create( |
| 199 |
[TwoFields(f1=i, f2=i + 1) for i in range(0, 1001)] |
| 200 |
) |
| 201 |
self.assertEqual(TwoFields.objects.count(), 1001) |
| 202 |
self.assertEqual( |
| 203 |
TwoFields.objects.filter(f1__gte=450, f1__lte=550).count(), 101 |
| 204 |
) |
| 205 |
self.assertEqual(TwoFields.objects.filter(f2__gte=901).count(), 101) |
| 206 |
|
| 207 |
@skipUnlessDBFeature("has_bulk_insert") |
| 208 |
def test_large_single_field_batch(self): |
| 209 |
# SQLite had a problem with more than 500 UNIONed selects in single |
| 210 |
# query. |
| 211 |
Restaurant.objects.bulk_create([Restaurant() for i in range(0, 501)]) |
| 212 |
|
| 213 |
@skipUnlessDBFeature("has_bulk_insert") |
| 214 |
def test_large_batch_efficiency(self): |
| 215 |
with override_settings(DEBUG=True): |
| 216 |
connection.queries_log.clear() |
| 217 |
TwoFields.objects.bulk_create( |
| 218 |
[TwoFields(f1=i, f2=i + 1) for i in range(0, 1001)] |
| 219 |
) |
| 220 |
self.assertLess(len(connection.queries), 10) |
| 221 |
|
| 222 |
def test_large_batch_mixed(self): |
| 223 |
""" |
| 224 |
Test inserting a large batch with objects having primary key set |
| 225 |
mixed together with objects without PK set. |
| 226 |
""" |
| 227 |
TwoFields.objects.bulk_create( |
| 228 |
[ |
| 229 |
TwoFields(id=i if i % 2 == 0 else None, f1=i, f2=i + 1) |
| 230 |
for i in range(100000, 101000) |
| 231 |
] |
| 232 |
) |
| 233 |
self.assertEqual(TwoFields.objects.count(), 1000) |
| 234 |
# We can't assume much about the ID's created, except that the above |
| 235 |
# created IDs must exist. |
| 236 |
id_range = range(100000, 101000, 2) |
| 237 |
self.assertEqual(TwoFields.objects.filter(id__in=id_range).count(), 500) |
| 238 |
self.assertEqual(TwoFields.objects.exclude(id__in=id_range).count(), 500) |
| 239 |
|
| 240 |
@skipUnlessDBFeature("has_bulk_insert") |
| 241 |
def test_large_batch_mixed_efficiency(self): |
| 242 |
""" |
| 243 |
Test inserting a large batch with objects having primary key set |
| 244 |
mixed together with objects without PK set. |
| 245 |
""" |
| 246 |
with override_settings(DEBUG=True): |
| 247 |
connection.queries_log.clear() |
| 248 |
TwoFields.objects.bulk_create( |
| 249 |
[ |
| 250 |
TwoFields(id=i if i % 2 == 0 else None, f1=i, f2=i + 1) |
| 251 |
for i in range(100000, 101000) |
| 252 |
] |
| 253 |
) |
| 254 |
self.assertLess(len(connection.queries), 10) |
| 255 |
|
| 256 |
def test_explicit_batch_size(self): |
| 257 |
objs = [TwoFields(f1=i, f2=i) for i in range(0, 4)] |
| 258 |
num_objs = len(objs) |
| 259 |
TwoFields.objects.bulk_create(objs, batch_size=1) |
| 260 |
self.assertEqual(TwoFields.objects.count(), num_objs) |
| 261 |
TwoFields.objects.all().delete() |
| 262 |
TwoFields.objects.bulk_create(objs, batch_size=2) |
| 263 |
self.assertEqual(TwoFields.objects.count(), num_objs) |
| 264 |
TwoFields.objects.all().delete() |
| 265 |
TwoFields.objects.bulk_create(objs, batch_size=3) |
| 266 |
self.assertEqual(TwoFields.objects.count(), num_objs) |
| 267 |
TwoFields.objects.all().delete() |
| 268 |
TwoFields.objects.bulk_create(objs, batch_size=num_objs) |
| 269 |
self.assertEqual(TwoFields.objects.count(), num_objs) |
| 270 |
|
| 271 |
def test_empty_model(self): |
| 272 |
NoFields.objects.bulk_create([NoFields() for i in range(2)]) |
| 273 |
self.assertEqual(NoFields.objects.count(), 2) |
| 274 |
|
| 275 |
@skipUnlessDBFeature("has_bulk_insert") |
| 276 |
def test_explicit_batch_size_efficiency(self): |
| 277 |
objs = [TwoFields(f1=i, f2=i) for i in range(0, 100)] |
| 278 |
with self.assertNumQueries(2): |
| 279 |
TwoFields.objects.bulk_create(objs, 50) |
| 280 |
TwoFields.objects.all().delete() |
| 281 |
with self.assertNumQueries(1): |
| 282 |
TwoFields.objects.bulk_create(objs, len(objs)) |
| 283 |
|
| 284 |
@skipUnlessDBFeature("has_bulk_insert") |
| 285 |
def test_explicit_batch_size_respects_max_batch_size(self): |
| 286 |
objs = [Country(name=f"Country {i}") for i in range(1000)] |
| 287 |
fields = ["name", "iso_two_letter", "description"] |
| 288 |
max_batch_size = max(connection.ops.bulk_batch_size(fields, objs), 1) |
| 289 |
with self.assertNumQueries(ceil(len(objs) / max_batch_size)): |
| 290 |
Country.objects.bulk_create(objs, batch_size=max_batch_size + 1) |
| 291 |
|
| 292 |
@skipUnlessDBFeature("has_bulk_insert") |
| 293 |
def test_bulk_insert_expressions(self): |
| 294 |
Restaurant.objects.bulk_create( |
| 295 |
[ |
| 296 |
Restaurant(name="Sam's Shake Shack"), |
| 297 |
Restaurant(name=Lower(Value("Betty's Beetroot Bar"))), |
| 298 |
] |
| 299 |
) |
| 300 |
bbb = Restaurant.objects.filter(name="betty's beetroot bar") |
| 301 |
self.assertEqual(bbb.count(), 1) |
| 302 |
|
| 303 |
@skipUnlessDBFeature("has_bulk_insert") |
| 304 |
def test_bulk_insert_now(self): |
| 305 |
NullableFields.objects.bulk_create( |
| 306 |
[ |
| 307 |
NullableFields(datetime_field=Now()), |
| 308 |
NullableFields(datetime_field=Now()), |
| 309 |
] |
| 310 |
) |
| 311 |
self.assertEqual( |
| 312 |
NullableFields.objects.filter(datetime_field__isnull=False).count(), |
| 313 |
2, |
| 314 |
) |
| 315 |
|
| 316 |
@skipUnlessDBFeature("has_bulk_insert") |
| 317 |
def test_bulk_insert_nullable_fields(self): |
| 318 |
fk_to_auto_fields = { |
| 319 |
"auto_field": NoFields.objects.create(), |
| 320 |
"small_auto_field": SmallAutoFieldModel.objects.create(), |
| 321 |
"big_auto_field": BigAutoFieldModel.objects.create(), |
| 322 |
} |
| 323 |
# NULL can be mixed with other values in nullable fields |
| 324 |
nullable_fields = [ |
| 325 |
field for field in NullableFields._meta.get_fields() if field.name != "id" |
| 326 |
] |
| 327 |
NullableFields.objects.bulk_create( |
| 328 |
[ |
| 329 |
NullableFields(**{**fk_to_auto_fields, field.name: None}) |
| 330 |
for field in nullable_fields |
| 331 |
] |
| 332 |
) |
| 333 |
self.assertEqual(NullableFields.objects.count(), len(nullable_fields)) |
| 334 |
for field in nullable_fields: |
| 335 |
with self.subTest(field=field): |
| 336 |
field_value = "" if isinstance(field, FileField) else None |
| 337 |
self.assertEqual( |
| 338 |
NullableFields.objects.filter(**{field.name: field_value}).count(), |
| 339 |
1, |
| 340 |
) |
| 341 |
|
| 342 |
@skipUnlessDBFeature("can_return_rows_from_bulk_insert") |
| 343 |
def test_set_pk_and_insert_single_item(self): |
| 344 |
with self.assertNumQueries(1): |
| 345 |
countries = Country.objects.bulk_create([self.data[0]]) |
| 346 |
self.assertEqual(len(countries), 1) |
| 347 |
self.assertEqual(Country.objects.get(pk=countries[0].pk), countries[0]) |
| 348 |
|
| 349 |
@skipUnlessDBFeature("can_return_rows_from_bulk_insert") |
| 350 |
def test_set_pk_and_query_efficiency(self): |
| 351 |
with self.assertNumQueries(1): |
| 352 |
countries = Country.objects.bulk_create(self.data) |
| 353 |
self.assertEqual(len(countries), 4) |
| 354 |
self.assertEqual(Country.objects.get(pk=countries[0].pk), countries[0]) |
| 355 |
self.assertEqual(Country.objects.get(pk=countries[1].pk), countries[1]) |
| 356 |
self.assertEqual(Country.objects.get(pk=countries[2].pk), countries[2]) |
| 357 |
self.assertEqual(Country.objects.get(pk=countries[3].pk), countries[3]) |
| 358 |
|
| 359 |
@skipUnlessDBFeature("can_return_rows_from_bulk_insert") |
| 360 |
def test_set_state(self): |
| 361 |
country_nl = Country(name="Netherlands", iso_two_letter="NL") |
| 362 |
country_be = Country(name="Belgium", iso_two_letter="BE") |
| 363 |
Country.objects.bulk_create([country_nl]) |
| 364 |
country_be.save() |
| 365 |
# Objects save via bulk_create() and save() should have equal state. |
| 366 |
self.assertEqual(country_nl._state.adding, country_be._state.adding) |
| 367 |
self.assertEqual(country_nl._state.db, country_be._state.db) |
| 368 |
|
| 369 |
def test_set_state_with_pk_specified(self): |
| 370 |
state_ca = State(two_letter_code="CA") |
| 371 |
state_ny = State(two_letter_code="NY") |
| 372 |
State.objects.bulk_create([state_ca]) |
| 373 |
state_ny.save() |
| 374 |
# Objects save via bulk_create() and save() should have equal state. |
| 375 |
self.assertEqual(state_ca._state.adding, state_ny._state.adding) |
| 376 |
self.assertEqual(state_ca._state.db, state_ny._state.db) |
| 377 |
|
| 378 |
@skipIfDBFeature("supports_ignore_conflicts") |
| 379 |
def test_ignore_conflicts_value_error(self): |
| 380 |
message = "This database backend does not support ignoring conflicts." |
| 381 |
with self.assertRaisesMessage(NotSupportedError, message): |
| 382 |
TwoFields.objects.bulk_create(self.data, ignore_conflicts=True) |
| 383 |
|
| 384 |
@skipUnlessDBFeature("supports_ignore_conflicts") |
| 385 |
def test_ignore_conflicts_ignore(self): |
| 386 |
data = [ |
| 387 |
TwoFields(f1=1, f2=1), |
| 388 |
TwoFields(f1=2, f2=2), |
| 389 |
TwoFields(f1=3, f2=3), |
| 390 |
] |
| 391 |
TwoFields.objects.bulk_create(data) |
| 392 |
self.assertEqual(TwoFields.objects.count(), 3) |
| 393 |
# With ignore_conflicts=True, conflicts are ignored. |
| 394 |
conflicting_objects = [ |
| 395 |
TwoFields(f1=2, f2=2), |
| 396 |
TwoFields(f1=3, f2=3), |
| 397 |
] |
| 398 |
TwoFields.objects.bulk_create([conflicting_objects[0]], ignore_conflicts=True) |
| 399 |
TwoFields.objects.bulk_create(conflicting_objects, ignore_conflicts=True) |
| 400 |
self.assertEqual(TwoFields.objects.count(), 3) |
| 401 |
self.assertIsNone(conflicting_objects[0].pk) |
| 402 |
self.assertIsNone(conflicting_objects[1].pk) |
| 403 |
# New objects are created and conflicts are ignored. |
| 404 |
new_object = TwoFields(f1=4, f2=4) |
| 405 |
TwoFields.objects.bulk_create( |
| 406 |
conflicting_objects + [new_object], ignore_conflicts=True |
| 407 |
) |
| 408 |
self.assertEqual(TwoFields.objects.count(), 4) |
| 409 |
self.assertIsNone(new_object.pk) |
| 410 |
# Without ignore_conflicts=True, there's a problem. |
| 411 |
with self.assertRaises(IntegrityError): |
| 412 |
TwoFields.objects.bulk_create(conflicting_objects) |
| 413 |
|
| 414 |
def test_nullable_fk_after_parent(self): |
| 415 |
parent = NoFields() |
| 416 |
child = NullableFields(auto_field=parent, integer_field=88) |
| 417 |
parent.save() |
| 418 |
NullableFields.objects.bulk_create([child]) |
| 419 |
child = NullableFields.objects.get(integer_field=88) |
| 420 |
self.assertEqual(child.auto_field, parent) |
| 421 |
|
| 422 |
@skipUnlessDBFeature("can_return_rows_from_bulk_insert") |
| 423 |
def test_nullable_fk_after_parent_bulk_create(self): |
| 424 |
parent = NoFields() |
| 425 |
child = NullableFields(auto_field=parent, integer_field=88) |
| 426 |
NoFields.objects.bulk_create([parent]) |
| 427 |
NullableFields.objects.bulk_create([child]) |
| 428 |
child = NullableFields.objects.get(integer_field=88) |
| 429 |
self.assertEqual(child.auto_field, parent) |
| 430 |
|
| 431 |
def test_unsaved_parent(self): |
| 432 |
parent = NoFields() |
| 433 |
msg = ( |
| 434 |
"bulk_create() prohibited to prevent data loss due to unsaved " |
| 435 |
"related object 'auto_field'." |
| 436 |
) |
| 437 |
with self.assertRaisesMessage(ValueError, msg): |
| 438 |
NullableFields.objects.bulk_create([NullableFields(auto_field=parent)]) |
| 439 |
|
| 440 |
def test_invalid_batch_size_exception(self): |
| 441 |
msg = "Batch size must be a positive integer." |
| 442 |
with self.assertRaisesMessage(ValueError, msg): |
| 443 |
Country.objects.bulk_create([], batch_size=-1) |
| 444 |
|
| 445 |
@skipIfDBFeature("supports_update_conflicts") |
| 446 |
def test_update_conflicts_unsupported(self): |
| 447 |
msg = "This database backend does not support updating conflicts." |
| 448 |
with self.assertRaisesMessage(NotSupportedError, msg): |
| 449 |
Country.objects.bulk_create(self.data, update_conflicts=True) |
| 450 |
|
| 451 |
@skipUnlessDBFeature("supports_ignore_conflicts", "supports_update_conflicts") |
| 452 |
def test_ignore_update_conflicts_exclusive(self): |
| 453 |
msg = "ignore_conflicts and update_conflicts are mutually exclusive" |
| 454 |
with self.assertRaisesMessage(ValueError, msg): |
| 455 |
Country.objects.bulk_create( |
| 456 |
self.data, |
| 457 |
ignore_conflicts=True, |
| 458 |
update_conflicts=True, |
| 459 |
) |
| 460 |
|
| 461 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 462 |
def test_update_conflicts_no_update_fields(self): |
| 463 |
msg = ( |
| 464 |
"Fields that will be updated when a row insertion fails on " |
| 465 |
"conflicts must be provided." |
| 466 |
) |
| 467 |
with self.assertRaisesMessage(ValueError, msg): |
| 468 |
Country.objects.bulk_create(self.data, update_conflicts=True) |
| 469 |
|
| 470 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 471 |
@skipIfDBFeature("supports_update_conflicts_with_target") |
| 472 |
def test_update_conflicts_unique_field_unsupported(self): |
| 473 |
msg = ( |
| 474 |
"This database backend does not support updating conflicts with " |
| 475 |
"specifying unique fields that can trigger the upsert." |
| 476 |
) |
| 477 |
with self.assertRaisesMessage(NotSupportedError, msg): |
| 478 |
TwoFields.objects.bulk_create( |
| 479 |
[TwoFields(f1=1, f2=1), TwoFields(f1=2, f2=2)], |
| 480 |
update_conflicts=True, |
| 481 |
update_fields=["f2"], |
| 482 |
unique_fields=["f1"], |
| 483 |
) |
| 484 |
|
| 485 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 486 |
def test_update_conflicts_nonexistent_update_fields(self): |
| 487 |
unique_fields = None |
| 488 |
if connection.features.supports_update_conflicts_with_target: |
| 489 |
unique_fields = ["f1"] |
| 490 |
msg = "TwoFields has no field named 'nonexistent'" |
| 491 |
with self.assertRaisesMessage(FieldDoesNotExist, msg): |
| 492 |
TwoFields.objects.bulk_create( |
| 493 |
[TwoFields(f1=1, f2=1), TwoFields(f1=2, f2=2)], |
| 494 |
update_conflicts=True, |
| 495 |
update_fields=["nonexistent"], |
| 496 |
unique_fields=unique_fields, |
| 497 |
) |
| 498 |
|
| 499 |
@skipUnlessDBFeature( |
| 500 |
"supports_update_conflicts", |
| 501 |
"supports_update_conflicts_with_target", |
| 502 |
) |
| 503 |
def test_update_conflicts_unique_fields_required(self): |
| 504 |
msg = "Unique fields that can trigger the upsert must be provided." |
| 505 |
with self.assertRaisesMessage(ValueError, msg): |
| 506 |
TwoFields.objects.bulk_create( |
| 507 |
[TwoFields(f1=1, f2=1), TwoFields(f1=2, f2=2)], |
| 508 |
update_conflicts=True, |
| 509 |
update_fields=["f1"], |
| 510 |
) |
| 511 |
|
| 512 |
@skipUnlessDBFeature( |
| 513 |
"supports_update_conflicts", |
| 514 |
"supports_update_conflicts_with_target", |
| 515 |
) |
| 516 |
def test_update_conflicts_invalid_update_fields(self): |
| 517 |
msg = "bulk_create() can only be used with concrete fields in update_fields." |
| 518 |
# Reverse one-to-one relationship. |
| 519 |
with self.assertRaisesMessage(ValueError, msg): |
| 520 |
Country.objects.bulk_create( |
| 521 |
self.data, |
| 522 |
update_conflicts=True, |
| 523 |
update_fields=["relatedmodel"], |
| 524 |
unique_fields=["pk"], |
| 525 |
) |
| 526 |
# Many-to-many relationship. |
| 527 |
with self.assertRaisesMessage(ValueError, msg): |
| 528 |
RelatedModel.objects.bulk_create( |
| 529 |
[RelatedModel(country=self.data[0])], |
| 530 |
update_conflicts=True, |
| 531 |
update_fields=["big_auto_fields"], |
| 532 |
unique_fields=["country"], |
| 533 |
) |
| 534 |
|
| 535 |
@skipUnlessDBFeature( |
| 536 |
"supports_update_conflicts", |
| 537 |
"supports_update_conflicts_with_target", |
| 538 |
) |
| 539 |
def test_update_conflicts_pk_in_update_fields(self): |
| 540 |
msg = "bulk_create() cannot be used with primary keys in update_fields." |
| 541 |
with self.assertRaisesMessage(ValueError, msg): |
| 542 |
BigAutoFieldModel.objects.bulk_create( |
| 543 |
[BigAutoFieldModel()], |
| 544 |
update_conflicts=True, |
| 545 |
update_fields=["id"], |
| 546 |
unique_fields=["id"], |
| 547 |
) |
| 548 |
|
| 549 |
@skipUnlessDBFeature( |
| 550 |
"supports_update_conflicts", |
| 551 |
"supports_update_conflicts_with_target", |
| 552 |
) |
| 553 |
def test_update_conflicts_invalid_unique_fields(self): |
| 554 |
msg = "bulk_create() can only be used with concrete fields in unique_fields." |
| 555 |
# Reverse one-to-one relationship. |
| 556 |
with self.assertRaisesMessage(ValueError, msg): |
| 557 |
Country.objects.bulk_create( |
| 558 |
self.data, |
| 559 |
update_conflicts=True, |
| 560 |
update_fields=["name"], |
| 561 |
unique_fields=["relatedmodel"], |
| 562 |
) |
| 563 |
# Many-to-many relationship. |
| 564 |
with self.assertRaisesMessage(ValueError, msg): |
| 565 |
RelatedModel.objects.bulk_create( |
| 566 |
[RelatedModel(country=self.data[0])], |
| 567 |
update_conflicts=True, |
| 568 |
update_fields=["name"], |
| 569 |
unique_fields=["big_auto_fields"], |
| 570 |
) |
| 571 |
|
| 572 |
def _test_update_conflicts_two_fields(self, unique_fields): |
| 573 |
TwoFields.objects.bulk_create( |
| 574 |
[ |
| 575 |
TwoFields(f1=1, f2=1, name="a"), |
| 576 |
TwoFields(f1=2, f2=2, name="b"), |
| 577 |
] |
| 578 |
) |
| 579 |
self.assertEqual(TwoFields.objects.count(), 2) |
| 580 |
|
| 581 |
conflicting_objects = [ |
| 582 |
TwoFields(f1=1, f2=1, name="c"), |
| 583 |
TwoFields(f1=2, f2=2, name="d"), |
| 584 |
] |
| 585 |
results = TwoFields.objects.bulk_create( |
| 586 |
conflicting_objects, |
| 587 |
update_conflicts=True, |
| 588 |
unique_fields=unique_fields, |
| 589 |
update_fields=["name"], |
| 590 |
) |
| 591 |
self.assertEqual(len(results), len(conflicting_objects)) |
| 592 |
if connection.features.can_return_rows_from_bulk_insert: |
| 593 |
for instance in results: |
| 594 |
self.assertIsNotNone(instance.pk) |
| 595 |
self.assertEqual(TwoFields.objects.count(), 2) |
| 596 |
self.assertCountEqual( |
| 597 |
TwoFields.objects.values("f1", "f2", "name"), |
| 598 |
[ |
| 599 |
{"f1": 1, "f2": 1, "name": "c"}, |
| 600 |
{"f1": 2, "f2": 2, "name": "d"}, |
| 601 |
], |
| 602 |
) |
| 603 |
|
| 604 |
@skipUnlessDBFeature( |
| 605 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 606 |
) |
| 607 |
def test_update_conflicts_two_fields_unique_fields_first(self): |
| 608 |
self._test_update_conflicts_two_fields(["f1"]) |
| 609 |
|
| 610 |
@skipUnlessDBFeature( |
| 611 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 612 |
) |
| 613 |
def test_update_conflicts_two_fields_unique_fields_second(self): |
| 614 |
self._test_update_conflicts_two_fields(["f2"]) |
| 615 |
|
| 616 |
@skipUnlessDBFeature( |
| 617 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 618 |
) |
| 619 |
def test_update_conflicts_unique_fields_pk(self): |
| 620 |
TwoFields.objects.bulk_create( |
| 621 |
[ |
| 622 |
TwoFields(f1=1, f2=1, name="a"), |
| 623 |
TwoFields(f1=2, f2=2, name="b"), |
| 624 |
] |
| 625 |
) |
| 626 |
|
| 627 |
obj1 = TwoFields.objects.get(f1=1) |
| 628 |
obj2 = TwoFields.objects.get(f1=2) |
| 629 |
conflicting_objects = [ |
| 630 |
TwoFields(pk=obj1.pk, f1=3, f2=3, name="c"), |
| 631 |
TwoFields(pk=obj2.pk, f1=4, f2=4, name="d"), |
| 632 |
] |
| 633 |
results = TwoFields.objects.bulk_create( |
| 634 |
conflicting_objects, |
| 635 |
update_conflicts=True, |
| 636 |
unique_fields=["pk"], |
| 637 |
update_fields=["name"], |
| 638 |
) |
| 639 |
self.assertEqual(len(results), len(conflicting_objects)) |
| 640 |
if connection.features.can_return_rows_from_bulk_insert: |
| 641 |
for instance in results: |
| 642 |
self.assertIsNotNone(instance.pk) |
| 643 |
self.assertEqual(TwoFields.objects.count(), 2) |
| 644 |
self.assertCountEqual( |
| 645 |
TwoFields.objects.values("f1", "f2", "name"), |
| 646 |
[ |
| 647 |
{"f1": 1, "f2": 1, "name": "c"}, |
| 648 |
{"f1": 2, "f2": 2, "name": "d"}, |
| 649 |
], |
| 650 |
) |
| 651 |
|
| 652 |
@skipUnlessDBFeature( |
| 653 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 654 |
) |
| 655 |
def test_update_conflicts_two_fields_unique_fields_both(self): |
| 656 |
with self.assertRaises((OperationalError, ProgrammingError)): |
| 657 |
self._test_update_conflicts_two_fields(["f1", "f2"]) |
| 658 |
|
| 659 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 660 |
@skipIfDBFeature("supports_update_conflicts_with_target") |
| 661 |
def test_update_conflicts_two_fields_no_unique_fields(self): |
| 662 |
self._test_update_conflicts_two_fields([]) |
| 663 |
|
| 664 |
def _test_update_conflicts_unique_two_fields(self, unique_fields): |
| 665 |
Country.objects.bulk_create(self.data) |
| 666 |
self.assertEqual(Country.objects.count(), 4) |
| 667 |
|
| 668 |
new_data = [ |
| 669 |
# Conflicting countries. |
| 670 |
Country( |
| 671 |
name="Germany", |
| 672 |
iso_two_letter="DE", |
| 673 |
description=("Germany is a country in Central Europe."), |
| 674 |
), |
| 675 |
Country( |
| 676 |
name="Czech Republic", |
| 677 |
iso_two_letter="CZ", |
| 678 |
description=( |
| 679 |
"The Czech Republic is a landlocked country in Central Europe." |
| 680 |
), |
| 681 |
), |
| 682 |
# New countries. |
| 683 |
Country(name="Australia", iso_two_letter="AU"), |
| 684 |
Country( |
| 685 |
name="Japan", |
| 686 |
iso_two_letter="JP", |
| 687 |
description=("Japan is an island country in East Asia."), |
| 688 |
), |
| 689 |
] |
| 690 |
results = Country.objects.bulk_create( |
| 691 |
new_data, |
| 692 |
update_conflicts=True, |
| 693 |
update_fields=["description"], |
| 694 |
unique_fields=unique_fields, |
| 695 |
) |
| 696 |
self.assertEqual(len(results), len(new_data)) |
| 697 |
if connection.features.can_return_rows_from_bulk_insert: |
| 698 |
for instance in results: |
| 699 |
self.assertIsNotNone(instance.pk) |
| 700 |
self.assertEqual(Country.objects.count(), 6) |
| 701 |
self.assertCountEqual( |
| 702 |
Country.objects.values("iso_two_letter", "description"), |
| 703 |
[ |
| 704 |
{"iso_two_letter": "US", "description": ""}, |
| 705 |
{"iso_two_letter": "NL", "description": ""}, |
| 706 |
{ |
| 707 |
"iso_two_letter": "DE", |
| 708 |
"description": ("Germany is a country in Central Europe."), |
| 709 |
}, |
| 710 |
{ |
| 711 |
"iso_two_letter": "CZ", |
| 712 |
"description": ( |
| 713 |
"The Czech Republic is a landlocked country in Central Europe." |
| 714 |
), |
| 715 |
}, |
| 716 |
{"iso_two_letter": "AU", "description": ""}, |
| 717 |
{ |
| 718 |
"iso_two_letter": "JP", |
| 719 |
"description": ("Japan is an island country in East Asia."), |
| 720 |
}, |
| 721 |
], |
| 722 |
) |
| 723 |
|
| 724 |
@skipUnlessDBFeature( |
| 725 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 726 |
) |
| 727 |
def test_update_conflicts_unique_two_fields_unique_fields_both(self): |
| 728 |
self._test_update_conflicts_unique_two_fields(["iso_two_letter", "name"]) |
| 729 |
|
| 730 |
@skipUnlessDBFeature( |
| 731 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 732 |
) |
| 733 |
def test_update_conflicts_unique_two_fields_unique_fields_one(self): |
| 734 |
with self.assertRaises((OperationalError, ProgrammingError)): |
| 735 |
self._test_update_conflicts_unique_two_fields(["iso_two_letter"]) |
| 736 |
|
| 737 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 738 |
@skipIfDBFeature("supports_update_conflicts_with_target") |
| 739 |
def test_update_conflicts_unique_two_fields_unique_no_unique_fields(self): |
| 740 |
self._test_update_conflicts_unique_two_fields([]) |
| 741 |
|
| 742 |
def _test_update_conflicts(self, unique_fields): |
| 743 |
UpsertConflict.objects.bulk_create( |
| 744 |
[ |
| 745 |
UpsertConflict(number=1, rank=1, name="John"), |
| 746 |
UpsertConflict(number=2, rank=2, name="Mary"), |
| 747 |
UpsertConflict(number=3, rank=3, name="Hannah"), |
| 748 |
] |
| 749 |
) |
| 750 |
self.assertEqual(UpsertConflict.objects.count(), 3) |
| 751 |
|
| 752 |
conflicting_objects = [ |
| 753 |
UpsertConflict(number=1, rank=4, name="Steve"), |
| 754 |
UpsertConflict(number=2, rank=2, name="Olivia"), |
| 755 |
UpsertConflict(number=3, rank=1, name="Hannah"), |
| 756 |
] |
| 757 |
results = UpsertConflict.objects.bulk_create( |
| 758 |
conflicting_objects, |
| 759 |
update_conflicts=True, |
| 760 |
update_fields=["name", "rank"], |
| 761 |
unique_fields=unique_fields, |
| 762 |
) |
| 763 |
self.assertEqual(len(results), len(conflicting_objects)) |
| 764 |
if connection.features.can_return_rows_from_bulk_insert: |
| 765 |
for instance in results: |
| 766 |
self.assertIsNotNone(instance.pk) |
| 767 |
self.assertEqual(UpsertConflict.objects.count(), 3) |
| 768 |
self.assertCountEqual( |
| 769 |
UpsertConflict.objects.values("number", "rank", "name"), |
| 770 |
[ |
| 771 |
{"number": 1, "rank": 4, "name": "Steve"}, |
| 772 |
{"number": 2, "rank": 2, "name": "Olivia"}, |
| 773 |
{"number": 3, "rank": 1, "name": "Hannah"}, |
| 774 |
], |
| 775 |
) |
| 776 |
|
| 777 |
results = UpsertConflict.objects.bulk_create( |
| 778 |
conflicting_objects + [UpsertConflict(number=4, rank=4, name="Mark")], |
| 779 |
update_conflicts=True, |
| 780 |
update_fields=["name", "rank"], |
| 781 |
unique_fields=unique_fields, |
| 782 |
) |
| 783 |
self.assertEqual(len(results), 4) |
| 784 |
if connection.features.can_return_rows_from_bulk_insert: |
| 785 |
for instance in results: |
| 786 |
self.assertIsNotNone(instance.pk) |
| 787 |
self.assertEqual(UpsertConflict.objects.count(), 4) |
| 788 |
self.assertCountEqual( |
| 789 |
UpsertConflict.objects.values("number", "rank", "name"), |
| 790 |
[ |
| 791 |
{"number": 1, "rank": 4, "name": "Steve"}, |
| 792 |
{"number": 2, "rank": 2, "name": "Olivia"}, |
| 793 |
{"number": 3, "rank": 1, "name": "Hannah"}, |
| 794 |
{"number": 4, "rank": 4, "name": "Mark"}, |
| 795 |
], |
| 796 |
) |
| 797 |
|
| 798 |
@skipUnlessDBFeature( |
| 799 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 800 |
) |
| 801 |
def test_update_conflicts_unique_fields(self): |
| 802 |
self._test_update_conflicts(unique_fields=["number"]) |
| 803 |
|
| 804 |
@skipUnlessDBFeature("supports_update_conflicts") |
| 805 |
@skipIfDBFeature("supports_update_conflicts_with_target") |
| 806 |
def test_update_conflicts_no_unique_fields(self): |
| 807 |
self._test_update_conflicts([]) |
| 808 |
|
| 809 |
@skipUnlessDBFeature( |
| 810 |
"supports_update_conflicts", "supports_update_conflicts_with_target" |
| 811 |
) |
| 812 |
def test_update_conflicts_unique_fields_update_fields_db_column(self): |
| 813 |
FieldsWithDbColumns.objects.bulk_create( |
| 814 |
[ |
| 815 |
FieldsWithDbColumns(rank=1, name="a"), |
| 816 |
FieldsWithDbColumns(rank=2, name="b"), |
| 817 |
] |
| 818 |
) |
| 819 |
self.assertEqual(FieldsWithDbColumns.objects.count(), 2) |
| 820 |
|
| 821 |
conflicting_objects = [ |
| 822 |
FieldsWithDbColumns(rank=1, name="c"), |
| 823 |
FieldsWithDbColumns(rank=2, name="d"), |
| 824 |
] |
| 825 |
results = FieldsWithDbColumns.objects.bulk_create( |
| 826 |
conflicting_objects, |
| 827 |
update_conflicts=True, |
| 828 |
unique_fields=["rank"], |
| 829 |
update_fields=["name"], |
| 830 |
) |
| 831 |
self.assertEqual(len(results), len(conflicting_objects)) |
| 832 |
if connection.features.can_return_rows_from_bulk_insert: |
| 833 |
for instance in results: |
| 834 |
self.assertIsNotNone(instance.pk) |
| 835 |
self.assertEqual(FieldsWithDbColumns.objects.count(), 2) |
| 836 |
self.assertCountEqual( |
| 837 |
FieldsWithDbColumns.objects.values("rank", "name"), |
| 838 |
[ |
| 839 |
{"rank": 1, "name": "c"}, |
| 840 |
{"rank": 2, "name": "d"}, |
| 841 |
], |
| 842 |
) |