| 1 |
import functools |
| 2 |
import glob |
| 3 |
import gzip |
| 4 |
import os |
| 5 |
import sys |
| 6 |
import warnings |
| 7 |
import zipfile |
| 8 |
from itertools import product |
| 9 |
|
| 10 |
from django.apps import apps |
| 11 |
from django.conf import settings |
| 12 |
from django.core import serializers |
| 13 |
from django.core.exceptions import ImproperlyConfigured |
| 14 |
from django.core.management.base import BaseCommand, CommandError |
| 15 |
from django.core.management.color import no_style |
| 16 |
from django.core.management.utils import parse_apps_and_model_labels |
| 17 |
from django.db import ( |
| 18 |
DEFAULT_DB_ALIAS, |
| 19 |
DatabaseError, |
| 20 |
IntegrityError, |
| 21 |
connections, |
| 22 |
router, |
| 23 |
transaction, |
| 24 |
) |
| 25 |
from django.utils.functional import cached_property |
| 26 |
|
| 27 |
try: |
| 28 |
import bz2 |
| 29 |
|
| 30 |
has_bz2 = True |
| 31 |
except ImportError: |
| 32 |
has_bz2 = False |
| 33 |
|
| 34 |
try: |
| 35 |
import lzma |
| 36 |
|
| 37 |
has_lzma = True |
| 38 |
except ImportError: |
| 39 |
has_lzma = False |
| 40 |
|
| 41 |
READ_STDIN = "-" |
| 42 |
|
| 43 |
|
| 44 |
class Command(BaseCommand): |
| 45 |
help = "Installs the named fixture(s) in the database." |
| 46 |
missing_args_message = ( |
| 47 |
"No database fixture specified. Please provide the path of at least " |
| 48 |
"one fixture in the command line." |
| 49 |
) |
| 50 |
|
| 51 |
def add_arguments(self, parser): |
| 52 |
parser.add_argument( |
| 53 |
"args", metavar="fixture", nargs="+", help="Fixture labels." |
| 54 |
) |
| 55 |
parser.add_argument( |
| 56 |
"--database", |
| 57 |
default=DEFAULT_DB_ALIAS, |
| 58 |
choices=tuple(connections), |
| 59 |
help=( |
| 60 |
"Nominates a specific database to load fixtures into. Defaults to the " |
| 61 |
'"default" database.' |
| 62 |
), |
| 63 |
) |
| 64 |
parser.add_argument( |
| 65 |
"--app", |
| 66 |
dest="app_label", |
| 67 |
help="Only look for fixtures in the specified app.", |
| 68 |
) |
| 69 |
parser.add_argument( |
| 70 |
"--ignorenonexistent", |
| 71 |
"-i", |
| 72 |
action="store_true", |
| 73 |
dest="ignore", |
| 74 |
help="Ignores entries in the serialized data for fields that do not " |
| 75 |
"currently exist on the model.", |
| 76 |
) |
| 77 |
parser.add_argument( |
| 78 |
"-e", |
| 79 |
"--exclude", |
| 80 |
action="append", |
| 81 |
default=[], |
| 82 |
help=( |
| 83 |
"An app_label or app_label.ModelName to exclude. Can be used multiple " |
| 84 |
"times." |
| 85 |
), |
| 86 |
) |
| 87 |
parser.add_argument( |
| 88 |
"--format", |
| 89 |
help="Format of serialized data when reading from stdin.", |
| 90 |
) |
| 91 |
|
| 92 |
def handle(self, *fixture_labels, **options): |
| 93 |
self.ignore = options["ignore"] |
| 94 |
self.using = options["database"] |
| 95 |
self.app_label = options["app_label"] |
| 96 |
self.verbosity = options["verbosity"] |
| 97 |
self.excluded_models, self.excluded_apps = parse_apps_and_model_labels( |
| 98 |
options["exclude"] |
| 99 |
) |
| 100 |
self.format = options["format"] |
| 101 |
|
| 102 |
with transaction.atomic(using=self.using): |
| 103 |
self.loaddata(fixture_labels) |
| 104 |
|
| 105 |
# Close the DB connection -- unless we're still in a transaction. This |
| 106 |
# is required as a workaround for an edge case in MySQL: if the same |
| 107 |
# connection is used to create tables, load data, and query, the query |
| 108 |
# can return incorrect results. See Django #7572, MySQL #37735. |
| 109 |
if transaction.get_autocommit(self.using): |
| 110 |
connections[self.using].close() |
| 111 |
|
| 112 |
@cached_property |
| 113 |
def compression_formats(self): |
| 114 |
"""A dict mapping format names to (open function, mode arg) tuples.""" |
| 115 |
# Forcing binary mode may be revisited after dropping Python 2 support |
| 116 |
# (see #22399). |
| 117 |
compression_formats = { |
| 118 |
None: (open, "rb"), |
| 119 |
"gz": (gzip.GzipFile, "rb"), |
| 120 |
"zip": (SingleZipReader, "r"), |
| 121 |
"stdin": (lambda *args: sys.stdin, None), |
| 122 |
} |
| 123 |
if has_bz2: |
| 124 |
compression_formats["bz2"] = (bz2.BZ2File, "r") |
| 125 |
if has_lzma: |
| 126 |
compression_formats["lzma"] = (lzma.LZMAFile, "r") |
| 127 |
compression_formats["xz"] = (lzma.LZMAFile, "r") |
| 128 |
return compression_formats |
| 129 |
|
| 130 |
def reset_sequences(self, connection, models): |
| 131 |
"""Reset database sequences for the given connection and models.""" |
| 132 |
sequence_sql = connection.ops.sequence_reset_sql(no_style(), models) |
| 133 |
if sequence_sql: |
| 134 |
if self.verbosity >= 2: |
| 135 |
self.stdout.write("Resetting sequences") |
| 136 |
with connection.cursor() as cursor: |
| 137 |
for line in sequence_sql: |
| 138 |
cursor.execute(line) |
| 139 |
|
| 140 |
def loaddata(self, fixture_labels): |
| 141 |
connection = connections[self.using] |
| 142 |
|
| 143 |
# Keep a count of the installed objects and fixtures |
| 144 |
self.fixture_count = 0 |
| 145 |
self.loaded_object_count = 0 |
| 146 |
self.fixture_object_count = 0 |
| 147 |
self.models = set() |
| 148 |
|
| 149 |
self.serialization_formats = serializers.get_public_serializer_formats() |
| 150 |
|
| 151 |
# Django's test suite repeatedly tries to load initial_data fixtures |
| 152 |
# from apps that don't have any fixtures. Because disabling constraint |
| 153 |
# checks can be expensive on some database (especially MSSQL), bail |
| 154 |
# out early if no fixtures are found. |
| 155 |
for fixture_label in fixture_labels: |
| 156 |
if self.find_fixtures(fixture_label): |
| 157 |
break |
| 158 |
else: |
| 159 |
return |
| 160 |
|
| 161 |
self.objs_with_deferred_fields = [] |
| 162 |
with connection.constraint_checks_disabled(): |
| 163 |
for fixture_label in fixture_labels: |
| 164 |
self.load_label(fixture_label) |
| 165 |
for obj in self.objs_with_deferred_fields: |
| 166 |
obj.save_deferred_fields(using=self.using) |
| 167 |
|
| 168 |
# Since we disabled constraint checks, we must manually check for |
| 169 |
# any invalid keys that might have been added |
| 170 |
table_names = [model._meta.db_table for model in self.models] |
| 171 |
try: |
| 172 |
connection.check_constraints(table_names=table_names) |
| 173 |
except Exception as e: |
| 174 |
e.args = ("Problem installing fixtures: %s" % e,) |
| 175 |
raise |
| 176 |
|
| 177 |
# If we found even one object in a fixture, we need to reset the |
| 178 |
# database sequences. |
| 179 |
if self.loaded_object_count > 0: |
| 180 |
self.reset_sequences(connection, self.models) |
| 181 |
|
| 182 |
if self.verbosity >= 1: |
| 183 |
if self.fixture_object_count == self.loaded_object_count: |
| 184 |
self.stdout.write( |
| 185 |
"Installed %d object(s) from %d fixture(s)" |
| 186 |
% (self.loaded_object_count, self.fixture_count) |
| 187 |
) |
| 188 |
else: |
| 189 |
self.stdout.write( |
| 190 |
"Installed %d object(s) (of %d) from %d fixture(s)" |
| 191 |
% ( |
| 192 |
self.loaded_object_count, |
| 193 |
self.fixture_object_count, |
| 194 |
self.fixture_count, |
| 195 |
) |
| 196 |
) |
| 197 |
|
| 198 |
def save_obj(self, obj): |
| 199 |
"""Save an object if permitted.""" |
| 200 |
if ( |
| 201 |
obj.object._meta.app_config in self.excluded_apps |
| 202 |
or type(obj.object) in self.excluded_models |
| 203 |
): |
| 204 |
return False |
| 205 |
saved = False |
| 206 |
if router.allow_migrate_model(self.using, obj.object.__class__): |
| 207 |
saved = True |
| 208 |
self.models.add(obj.object.__class__) |
| 209 |
try: |
| 210 |
obj.save(using=self.using) |
| 211 |
# psycopg raises ValueError if data contains NUL chars. |
| 212 |
except (DatabaseError, IntegrityError, ValueError) as e: |
| 213 |
e.args = ( |
| 214 |
"Could not load %(object_label)s(pk=%(pk)s): %(error_msg)s" |
| 215 |
% { |
| 216 |
"object_label": obj.object._meta.label, |
| 217 |
"pk": obj.object.pk, |
| 218 |
"error_msg": e, |
| 219 |
}, |
| 220 |
) |
| 221 |
raise |
| 222 |
if obj.deferred_fields: |
| 223 |
self.objs_with_deferred_fields.append(obj) |
| 224 |
return saved |
| 225 |
|
| 226 |
def load_label(self, fixture_label): |
| 227 |
"""Load fixtures files for a given label.""" |
| 228 |
show_progress = self.verbosity >= 3 |
| 229 |
for fixture_file, fixture_dir, fixture_name in self.find_fixtures( |
| 230 |
fixture_label |
| 231 |
): |
| 232 |
_, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file)) |
| 233 |
open_method, mode = self.compression_formats[cmp_fmt] |
| 234 |
fixture = open_method(fixture_file, mode) |
| 235 |
self.fixture_count += 1 |
| 236 |
objects_in_fixture = 0 |
| 237 |
loaded_objects_in_fixture = 0 |
| 238 |
if self.verbosity >= 2: |
| 239 |
self.stdout.write( |
| 240 |
"Installing %s fixture '%s' from %s." |
| 241 |
% (ser_fmt, fixture_name, humanize(fixture_dir)) |
| 242 |
) |
| 243 |
try: |
| 244 |
objects = serializers.deserialize( |
| 245 |
ser_fmt, |
| 246 |
fixture, |
| 247 |
using=self.using, |
| 248 |
ignorenonexistent=self.ignore, |
| 249 |
handle_forward_references=True, |
| 250 |
) |
| 251 |
|
| 252 |
for obj in objects: |
| 253 |
objects_in_fixture += 1 |
| 254 |
if self.save_obj(obj): |
| 255 |
loaded_objects_in_fixture += 1 |
| 256 |
if show_progress: |
| 257 |
self.stdout.write( |
| 258 |
"\rProcessed %i object(s)." % loaded_objects_in_fixture, |
| 259 |
ending="", |
| 260 |
) |
| 261 |
except Exception as e: |
| 262 |
if not isinstance(e, CommandError): |
| 263 |
e.args = ( |
| 264 |
"Problem installing fixture '%s': %s" % (fixture_file, e), |
| 265 |
) |
| 266 |
raise |
| 267 |
finally: |
| 268 |
fixture.close() |
| 269 |
|
| 270 |
if objects_in_fixture and show_progress: |
| 271 |
self.stdout.write() # Add a newline after progress indicator. |
| 272 |
self.loaded_object_count += loaded_objects_in_fixture |
| 273 |
self.fixture_object_count += objects_in_fixture |
| 274 |
# Warn if the fixture we loaded contains 0 objects. |
| 275 |
if objects_in_fixture == 0: |
| 276 |
warnings.warn( |
| 277 |
"No fixture data found for '%s'. (File format may be " |
| 278 |
"invalid.)" % fixture_name, |
| 279 |
RuntimeWarning, |
| 280 |
) |
| 281 |
|
| 282 |
def get_fixture_name_and_dirs(self, fixture_name): |
| 283 |
dirname, basename = os.path.split(fixture_name) |
| 284 |
if os.path.isabs(fixture_name): |
| 285 |
fixture_dirs = [dirname] |
| 286 |
else: |
| 287 |
fixture_dirs = self.fixture_dirs |
| 288 |
if os.path.sep in os.path.normpath(fixture_name): |
| 289 |
fixture_dirs = [os.path.join(dir_, dirname) for dir_ in fixture_dirs] |
| 290 |
return basename, fixture_dirs |
| 291 |
|
| 292 |
def get_targets(self, fixture_name, ser_fmt, cmp_fmt): |
| 293 |
databases = [self.using, None] |
| 294 |
cmp_fmts = self.compression_formats if cmp_fmt is None else [cmp_fmt] |
| 295 |
ser_fmts = self.serialization_formats if ser_fmt is None else [ser_fmt] |
| 296 |
return { |
| 297 |
"%s.%s" |
| 298 |
% ( |
| 299 |
fixture_name, |
| 300 |
".".join([ext for ext in combo if ext]), |
| 301 |
) |
| 302 |
for combo in product(databases, ser_fmts, cmp_fmts) |
| 303 |
} |
| 304 |
|
| 305 |
def find_fixture_files_in_dir(self, fixture_dir, fixture_name, targets): |
| 306 |
fixture_files_in_dir = [] |
| 307 |
path = os.path.join(fixture_dir, fixture_name) |
| 308 |
for candidate in glob.iglob(glob.escape(path) + "*"): |
| 309 |
if os.path.basename(candidate) in targets: |
| 310 |
# Save the fixture_dir and fixture_name for future error |
| 311 |
# messages. |
| 312 |
fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) |
| 313 |
return fixture_files_in_dir |
| 314 |
|
| 315 |
@functools.cache |
| 316 |
def find_fixtures(self, fixture_label): |
| 317 |
"""Find fixture files for a given label.""" |
| 318 |
if fixture_label == READ_STDIN: |
| 319 |
return [(READ_STDIN, None, READ_STDIN)] |
| 320 |
|
| 321 |
fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) |
| 322 |
if self.verbosity >= 2: |
| 323 |
self.stdout.write("Loading '%s' fixtures..." % fixture_name) |
| 324 |
|
| 325 |
fixture_name, fixture_dirs = self.get_fixture_name_and_dirs(fixture_name) |
| 326 |
targets = self.get_targets(fixture_name, ser_fmt, cmp_fmt) |
| 327 |
fixture_files = [] |
| 328 |
for fixture_dir in fixture_dirs: |
| 329 |
if self.verbosity >= 2: |
| 330 |
self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) |
| 331 |
fixture_files_in_dir = self.find_fixture_files_in_dir( |
| 332 |
fixture_dir, |
| 333 |
fixture_name, |
| 334 |
targets, |
| 335 |
) |
| 336 |
if self.verbosity >= 2 and not fixture_files_in_dir: |
| 337 |
self.stdout.write( |
| 338 |
"No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir)) |
| 339 |
) |
| 340 |
|
| 341 |
# Check kept for backwards-compatibility; it isn't clear why |
| 342 |
# duplicates are only allowed in different directories. |
| 343 |
if len(fixture_files_in_dir) > 1: |
| 344 |
raise CommandError( |
| 345 |
"Multiple fixtures named '%s' in %s. Aborting." |
| 346 |
% (fixture_name, humanize(fixture_dir)) |
| 347 |
) |
| 348 |
fixture_files.extend(fixture_files_in_dir) |
| 349 |
|
| 350 |
if not fixture_files: |
| 351 |
raise CommandError("No fixture named '%s' found." % fixture_name) |
| 352 |
|
| 353 |
return fixture_files |
| 354 |
|
| 355 |
@cached_property |
| 356 |
def fixture_dirs(self): |
| 357 |
""" |
| 358 |
Return a list of fixture directories. |
| 359 |
|
| 360 |
The list contains the 'fixtures' subdirectory of each installed |
| 361 |
application, if it exists, the directories in FIXTURE_DIRS, and the |
| 362 |
current directory. |
| 363 |
""" |
| 364 |
dirs = [] |
| 365 |
fixture_dirs = settings.FIXTURE_DIRS |
| 366 |
if len(fixture_dirs) != len(set(fixture_dirs)): |
| 367 |
raise ImproperlyConfigured("settings.FIXTURE_DIRS contains duplicates.") |
| 368 |
for app_config in apps.get_app_configs(): |
| 369 |
app_label = app_config.label |
| 370 |
app_dir = os.path.join(app_config.path, "fixtures") |
| 371 |
if app_dir in [str(d) for d in fixture_dirs]: |
| 372 |
raise ImproperlyConfigured( |
| 373 |
"'%s' is a default fixture directory for the '%s' app " |
| 374 |
"and cannot be listed in settings.FIXTURE_DIRS." |
| 375 |
% (app_dir, app_label) |
| 376 |
) |
| 377 |
|
| 378 |
if self.app_label and app_label != self.app_label: |
| 379 |
continue |
| 380 |
if os.path.isdir(app_dir): |
| 381 |
dirs.append(app_dir) |
| 382 |
dirs.extend(fixture_dirs) |
| 383 |
dirs.append("") |
| 384 |
return [os.path.realpath(d) for d in dirs] |
| 385 |
|
| 386 |
def parse_name(self, fixture_name): |
| 387 |
""" |
| 388 |
Split fixture name in name, serialization format, compression format. |
| 389 |
""" |
| 390 |
if fixture_name == READ_STDIN: |
| 391 |
if not self.format: |
| 392 |
raise CommandError( |
| 393 |
"--format must be specified when reading from stdin." |
| 394 |
) |
| 395 |
return READ_STDIN, self.format, "stdin" |
| 396 |
|
| 397 |
parts = fixture_name.rsplit(".", 2) |
| 398 |
|
| 399 |
if len(parts) > 1 and parts[-1] in self.compression_formats: |
| 400 |
cmp_fmt = parts[-1] |
| 401 |
parts = parts[:-1] |
| 402 |
else: |
| 403 |
cmp_fmt = None |
| 404 |
|
| 405 |
if len(parts) > 1: |
| 406 |
if parts[-1] in self.serialization_formats: |
| 407 |
ser_fmt = parts[-1] |
| 408 |
parts = parts[:-1] |
| 409 |
else: |
| 410 |
raise CommandError( |
| 411 |
"Problem installing fixture '%s': %s is not a known " |
| 412 |
"serialization format." % (".".join(parts[:-1]), parts[-1]) |
| 413 |
) |
| 414 |
else: |
| 415 |
ser_fmt = None |
| 416 |
|
| 417 |
name = ".".join(parts) |
| 418 |
|
| 419 |
return name, ser_fmt, cmp_fmt |
| 420 |
|
| 421 |
|
| 422 |
class SingleZipReader(zipfile.ZipFile): |
| 423 |
def __init__(self, *args, **kwargs): |
| 424 |
super().__init__(*args, **kwargs) |
| 425 |
if len(self.namelist()) != 1: |
| 426 |
raise ValueError("Zip-compressed fixtures must contain one file.") |
| 427 |
|
| 428 |
def read(self): |
| 429 |
return zipfile.ZipFile.read(self, self.namelist()[0]) |
| 430 |
|
| 431 |
|
| 432 |
def humanize(dirname): |
| 433 |
return "'%s'" % dirname if dirname else "absolute path" |