Debian Codegraph @

django/core/management/commands/loaddata.py

raw
1 import functools
2 import glob
3 import gzip
4 import os
5 import sys
6 import warnings
7 import zipfile
8 from itertools import product
9
10 from django.apps import apps
11 from django.conf import settings
12 from django.core import serializers
13 from django.core.exceptions import ImproperlyConfigured
14 from django.core.management.base import BaseCommand, CommandError
15 from django.core.management.color import no_style
16 from django.core.management.utils import parse_apps_and_model_labels
17 from django.db import (
18 DEFAULT_DB_ALIAS,
19 DatabaseError,
20 IntegrityError,
21 connections,
22 router,
23 transaction,
24 )
25 from django.utils.functional import cached_property
26
27 try:
28 import bz2
29
30 has_bz2 = True
31 except ImportError:
32 has_bz2 = False
33
34 try:
35 import lzma
36
37 has_lzma = True
38 except ImportError:
39 has_lzma = False
40
41 READ_STDIN = "-"
42
43
44 class Command(BaseCommand):
45 help = "Installs the named fixture(s) in the database."
46 missing_args_message = (
47 "No database fixture specified. Please provide the path of at least "
48 "one fixture in the command line."
49 )
50
51 def add_arguments(self, parser):
52 parser.add_argument(
53 "args", metavar="fixture", nargs="+", help="Fixture labels."
54 )
55 parser.add_argument(
56 "--database",
57 default=DEFAULT_DB_ALIAS,
58 choices=tuple(connections),
59 help=(
60 "Nominates a specific database to load fixtures into. Defaults to the "
61 '"default" database.'
62 ),
63 )
64 parser.add_argument(
65 "--app",
66 dest="app_label",
67 help="Only look for fixtures in the specified app.",
68 )
69 parser.add_argument(
70 "--ignorenonexistent",
71 "-i",
72 action="store_true",
73 dest="ignore",
74 help="Ignores entries in the serialized data for fields that do not "
75 "currently exist on the model.",
76 )
77 parser.add_argument(
78 "-e",
79 "--exclude",
80 action="append",
81 default=[],
82 help=(
83 "An app_label or app_label.ModelName to exclude. Can be used multiple "
84 "times."
85 ),
86 )
87 parser.add_argument(
88 "--format",
89 help="Format of serialized data when reading from stdin.",
90 )
91
92 def handle(self, *fixture_labels, **options):
93 self.ignore = options["ignore"]
94 self.using = options["database"]
95 self.app_label = options["app_label"]
96 self.verbosity = options["verbosity"]
97 self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(
98 options["exclude"]
99 )
100 self.format = options["format"]
101
102 with transaction.atomic(using=self.using):
103 self.loaddata(fixture_labels)
104
105 # Close the DB connection -- unless we're still in a transaction. This
106 # is required as a workaround for an edge case in MySQL: if the same
107 # connection is used to create tables, load data, and query, the query
108 # can return incorrect results. See Django #7572, MySQL #37735.
109 if transaction.get_autocommit(self.using):
110 connections[self.using].close()
111
112 @cached_property
113 def compression_formats(self):
114 """A dict mapping format names to (open function, mode arg) tuples."""
115 # Forcing binary mode may be revisited after dropping Python 2 support
116 # (see #22399).
117 compression_formats = {
118 None: (open, "rb"),
119 "gz": (gzip.GzipFile, "rb"),
120 "zip": (SingleZipReader, "r"),
121 "stdin": (lambda *args: sys.stdin, None),
122 }
123 if has_bz2:
124 compression_formats["bz2"] = (bz2.BZ2File, "r")
125 if has_lzma:
126 compression_formats["lzma"] = (lzma.LZMAFile, "r")
127 compression_formats["xz"] = (lzma.LZMAFile, "r")
128 return compression_formats
129
130 def reset_sequences(self, connection, models):
131 """Reset database sequences for the given connection and models."""
132 sequence_sql = connection.ops.sequence_reset_sql(no_style(), models)
133 if sequence_sql:
134 if self.verbosity >= 2:
135 self.stdout.write("Resetting sequences")
136 with connection.cursor() as cursor:
137 for line in sequence_sql:
138 cursor.execute(line)
139
140 def loaddata(self, fixture_labels):
141 connection = connections[self.using]
142
143 # Keep a count of the installed objects and fixtures
144 self.fixture_count = 0
145 self.loaded_object_count = 0
146 self.fixture_object_count = 0
147 self.models = set()
148
149 self.serialization_formats = serializers.get_public_serializer_formats()
150
151 # Django's test suite repeatedly tries to load initial_data fixtures
152 # from apps that don't have any fixtures. Because disabling constraint
153 # checks can be expensive on some database (especially MSSQL), bail
154 # out early if no fixtures are found.
155 for fixture_label in fixture_labels:
156 if self.find_fixtures(fixture_label):
157 break
158 else:
159 return
160
161 self.objs_with_deferred_fields = []
162 with connection.constraint_checks_disabled():
163 for fixture_label in fixture_labels:
164 self.load_label(fixture_label)
165 for obj in self.objs_with_deferred_fields:
166 obj.save_deferred_fields(using=self.using)
167
168 # Since we disabled constraint checks, we must manually check for
169 # any invalid keys that might have been added
170 table_names = [model._meta.db_table for model in self.models]
171 try:
172 connection.check_constraints(table_names=table_names)
173 except Exception as e:
174 e.args = ("Problem installing fixtures: %s" % e,)
175 raise
176
177 # If we found even one object in a fixture, we need to reset the
178 # database sequences.
179 if self.loaded_object_count > 0:
180 self.reset_sequences(connection, self.models)
181
182 if self.verbosity >= 1:
183 if self.fixture_object_count == self.loaded_object_count:
184 self.stdout.write(
185 "Installed %d object(s) from %d fixture(s)"
186 % (self.loaded_object_count, self.fixture_count)
187 )
188 else:
189 self.stdout.write(
190 "Installed %d object(s) (of %d) from %d fixture(s)"
191 % (
192 self.loaded_object_count,
193 self.fixture_object_count,
194 self.fixture_count,
195 )
196 )
197
198 def save_obj(self, obj):
199 """Save an object if permitted."""
200 if (
201 obj.object._meta.app_config in self.excluded_apps
202 or type(obj.object) in self.excluded_models
203 ):
204 return False
205 saved = False
206 if router.allow_migrate_model(self.using, obj.object.__class__):
207 saved = True
208 self.models.add(obj.object.__class__)
209 try:
210 obj.save(using=self.using)
211 # psycopg raises ValueError if data contains NUL chars.
212 except (DatabaseError, IntegrityError, ValueError) as e:
213 e.args = (
214 "Could not load %(object_label)s(pk=%(pk)s): %(error_msg)s"
215 % {
216 "object_label": obj.object._meta.label,
217 "pk": obj.object.pk,
218 "error_msg": e,
219 },
220 )
221 raise
222 if obj.deferred_fields:
223 self.objs_with_deferred_fields.append(obj)
224 return saved
225
226 def load_label(self, fixture_label):
227 """Load fixtures files for a given label."""
228 show_progress = self.verbosity >= 3
229 for fixture_file, fixture_dir, fixture_name in self.find_fixtures(
230 fixture_label
231 ):
232 _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
233 open_method, mode = self.compression_formats[cmp_fmt]
234 fixture = open_method(fixture_file, mode)
235 self.fixture_count += 1
236 objects_in_fixture = 0
237 loaded_objects_in_fixture = 0
238 if self.verbosity >= 2:
239 self.stdout.write(
240 "Installing %s fixture '%s' from %s."
241 % (ser_fmt, fixture_name, humanize(fixture_dir))
242 )
243 try:
244 objects = serializers.deserialize(
245 ser_fmt,
246 fixture,
247 using=self.using,
248 ignorenonexistent=self.ignore,
249 handle_forward_references=True,
250 )
251
252 for obj in objects:
253 objects_in_fixture += 1
254 if self.save_obj(obj):
255 loaded_objects_in_fixture += 1
256 if show_progress:
257 self.stdout.write(
258 "\rProcessed %i object(s)." % loaded_objects_in_fixture,
259 ending="",
260 )
261 except Exception as e:
262 if not isinstance(e, CommandError):
263 e.args = (
264 "Problem installing fixture '%s': %s" % (fixture_file, e),
265 )
266 raise
267 finally:
268 fixture.close()
269
270 if objects_in_fixture and show_progress:
271 self.stdout.write() # Add a newline after progress indicator.
272 self.loaded_object_count += loaded_objects_in_fixture
273 self.fixture_object_count += objects_in_fixture
274 # Warn if the fixture we loaded contains 0 objects.
275 if objects_in_fixture == 0:
276 warnings.warn(
277 "No fixture data found for '%s'. (File format may be "
278 "invalid.)" % fixture_name,
279 RuntimeWarning,
280 )
281
282 def get_fixture_name_and_dirs(self, fixture_name):
283 dirname, basename = os.path.split(fixture_name)
284 if os.path.isabs(fixture_name):
285 fixture_dirs = [dirname]
286 else:
287 fixture_dirs = self.fixture_dirs
288 if os.path.sep in os.path.normpath(fixture_name):
289 fixture_dirs = [os.path.join(dir_, dirname) for dir_ in fixture_dirs]
290 return basename, fixture_dirs
291
292 def get_targets(self, fixture_name, ser_fmt, cmp_fmt):
293 databases = [self.using, None]
294 cmp_fmts = self.compression_formats if cmp_fmt is None else [cmp_fmt]
295 ser_fmts = self.serialization_formats if ser_fmt is None else [ser_fmt]
296 return {
297 "%s.%s"
298 % (
299 fixture_name,
300 ".".join([ext for ext in combo if ext]),
301 )
302 for combo in product(databases, ser_fmts, cmp_fmts)
303 }
304
305 def find_fixture_files_in_dir(self, fixture_dir, fixture_name, targets):
306 fixture_files_in_dir = []
307 path = os.path.join(fixture_dir, fixture_name)
308 for candidate in glob.iglob(glob.escape(path) + "*"):
309 if os.path.basename(candidate) in targets:
310 # Save the fixture_dir and fixture_name for future error
311 # messages.
312 fixture_files_in_dir.append((candidate, fixture_dir, fixture_name))
313 return fixture_files_in_dir
314
315 @functools.cache
316 def find_fixtures(self, fixture_label):
317 """Find fixture files for a given label."""
318 if fixture_label == READ_STDIN:
319 return [(READ_STDIN, None, READ_STDIN)]
320
321 fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label)
322 if self.verbosity >= 2:
323 self.stdout.write("Loading '%s' fixtures..." % fixture_name)
324
325 fixture_name, fixture_dirs = self.get_fixture_name_and_dirs(fixture_name)
326 targets = self.get_targets(fixture_name, ser_fmt, cmp_fmt)
327 fixture_files = []
328 for fixture_dir in fixture_dirs:
329 if self.verbosity >= 2:
330 self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir))
331 fixture_files_in_dir = self.find_fixture_files_in_dir(
332 fixture_dir,
333 fixture_name,
334 targets,
335 )
336 if self.verbosity >= 2 and not fixture_files_in_dir:
337 self.stdout.write(
338 "No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))
339 )
340
341 # Check kept for backwards-compatibility; it isn't clear why
342 # duplicates are only allowed in different directories.
343 if len(fixture_files_in_dir) > 1:
344 raise CommandError(
345 "Multiple fixtures named '%s' in %s. Aborting."
346 % (fixture_name, humanize(fixture_dir))
347 )
348 fixture_files.extend(fixture_files_in_dir)
349
350 if not fixture_files:
351 raise CommandError("No fixture named '%s' found." % fixture_name)
352
353 return fixture_files
354
355 @cached_property
356 def fixture_dirs(self):
357 """
358 Return a list of fixture directories.
359
360 The list contains the 'fixtures' subdirectory of each installed
361 application, if it exists, the directories in FIXTURE_DIRS, and the
362 current directory.
363 """
364 dirs = []
365 fixture_dirs = settings.FIXTURE_DIRS
366 if len(fixture_dirs) != len(set(fixture_dirs)):
367 raise ImproperlyConfigured("settings.FIXTURE_DIRS contains duplicates.")
368 for app_config in apps.get_app_configs():
369 app_label = app_config.label
370 app_dir = os.path.join(app_config.path, "fixtures")
371 if app_dir in [str(d) for d in fixture_dirs]:
372 raise ImproperlyConfigured(
373 "'%s' is a default fixture directory for the '%s' app "
374 "and cannot be listed in settings.FIXTURE_DIRS."
375 % (app_dir, app_label)
376 )
377
378 if self.app_label and app_label != self.app_label:
379 continue
380 if os.path.isdir(app_dir):
381 dirs.append(app_dir)
382 dirs.extend(fixture_dirs)
383 dirs.append("")
384 return [os.path.realpath(d) for d in dirs]
385
386 def parse_name(self, fixture_name):
387 """
388 Split fixture name in name, serialization format, compression format.
389 """
390 if fixture_name == READ_STDIN:
391 if not self.format:
392 raise CommandError(
393 "--format must be specified when reading from stdin."
394 )
395 return READ_STDIN, self.format, "stdin"
396
397 parts = fixture_name.rsplit(".", 2)
398
399 if len(parts) > 1 and parts[-1] in self.compression_formats:
400 cmp_fmt = parts[-1]
401 parts = parts[:-1]
402 else:
403 cmp_fmt = None
404
405 if len(parts) > 1:
406 if parts[-1] in self.serialization_formats:
407 ser_fmt = parts[-1]
408 parts = parts[:-1]
409 else:
410 raise CommandError(
411 "Problem installing fixture '%s': %s is not a known "
412 "serialization format." % (".".join(parts[:-1]), parts[-1])
413 )
414 else:
415 ser_fmt = None
416
417 name = ".".join(parts)
418
419 return name, ser_fmt, cmp_fmt
420
421
422 class SingleZipReader(zipfile.ZipFile):
423 def __init__(self, *args, **kwargs):
424 super().__init__(*args, **kwargs)
425 if len(self.namelist()) != 1:
426 raise ValueError("Zip-compressed fixtures must contain one file.")
427
428 def read(self):
429 return zipfile.ZipFile.read(self, self.namelist()[0])
430
431
432 def humanize(dirname):
433 return "'%s'" % dirname if dirname else "absolute path"