Debian Codegraph @

django/contrib/staticfiles/management/commands/collectstatic.py

raw
1 import os
2
3 from django.apps import apps
4 from django.contrib.staticfiles.finders import get_finders
5 from django.contrib.staticfiles.storage import staticfiles_storage
6 from django.core.checks import Tags
7 from django.core.files.storage import FileSystemStorage
8 from django.core.management.base import BaseCommand, CommandError
9 from django.core.management.color import no_style
10 from django.utils.functional import cached_property
11
12
13 class Command(BaseCommand):
14 """
15 Copies or symlinks static files from different locations to the
16 settings.STATIC_ROOT.
17 """
18
19 help = "Collect static files in a single location."
20 requires_system_checks = [Tags.staticfiles]
21
22 def __init__(self, *args, **kwargs):
23 super().__init__(*args, **kwargs)
24 self.copied_files = []
25 self.symlinked_files = []
26 self.unmodified_files = []
27 self.post_processed_files = []
28 self.storage = staticfiles_storage
29 self.style = no_style()
30
31 @cached_property
32 def local(self):
33 try:
34 self.storage.path("")
35 except NotImplementedError:
36 return False
37 return True
38
39 def add_arguments(self, parser):
40 parser.add_argument(
41 "--noinput",
42 "--no-input",
43 action="store_false",
44 dest="interactive",
45 help="Do NOT prompt the user for input of any kind.",
46 )
47 parser.add_argument(
48 "--no-post-process",
49 action="store_false",
50 dest="post_process",
51 help="Do NOT post process collected files.",
52 )
53 parser.add_argument(
54 "-i",
55 "--ignore",
56 action="append",
57 default=[],
58 dest="ignore_patterns",
59 metavar="PATTERN",
60 help="Ignore files or directories matching this glob-style "
61 "pattern. Use multiple times to ignore more.",
62 )
63 parser.add_argument(
64 "-n",
65 "--dry-run",
66 action="store_true",
67 help="Do everything except modify the filesystem.",
68 )
69 parser.add_argument(
70 "-c",
71 "--clear",
72 action="store_true",
73 help="Clear the existing files using the storage "
74 "before trying to copy or link the original file.",
75 )
76 parser.add_argument(
77 "-l",
78 "--link",
79 action="store_true",
80 help="Create a symbolic link to each file instead of copying.",
81 )
82 parser.add_argument(
83 "--no-default-ignore",
84 action="store_false",
85 dest="use_default_ignore_patterns",
86 help=(
87 "Don't ignore the common private glob-style patterns (defaults to "
88 "'CVS', '.*' and '*~')."
89 ),
90 )
91
92 def set_options(self, **options):
93 """
94 Set instance variables based on an options dict
95 """
96 self.interactive = options["interactive"]
97 self.verbosity = options["verbosity"]
98 self.symlink = options["link"]
99 self.clear = options["clear"]
100 self.dry_run = options["dry_run"]
101 ignore_patterns = options["ignore_patterns"]
102 if options["use_default_ignore_patterns"]:
103 ignore_patterns += apps.get_app_config("staticfiles").ignore_patterns
104 self.ignore_patterns = list({os.path.normpath(p) for p in ignore_patterns})
105 self.post_process = options["post_process"]
106
107 def collect(self):
108 """
109 Perform the bulk of the work of collectstatic.
110
111 Split off from handle() to facilitate testing.
112 """
113 if self.symlink and not self.local:
114 raise CommandError("Can't symlink to a remote destination.")
115
116 if self.clear:
117 self.clear_dir("")
118
119 if self.symlink:
120 handler = self.link_file
121 else:
122 handler = self.copy_file
123
124 found_files = {}
125 for finder in get_finders():
126 for path, storage in finder.list(self.ignore_patterns):
127 # Prefix the relative path if the source storage contains it
128 if getattr(storage, "prefix", None):
129 prefixed_path = os.path.join(storage.prefix, path)
130 else:
131 prefixed_path = path
132
133 if prefixed_path not in found_files:
134 found_files[prefixed_path] = (storage, path)
135 handler(path, prefixed_path, storage)
136 else:
137 self.log(
138 "Found another file with the destination path '%s'. It "
139 "will be ignored since only the first encountered file "
140 "is collected. If this is not what you want, make sure "
141 "every static file has a unique path." % prefixed_path,
142 level=1,
143 )
144
145 # Storage backends may define a post_process() method.
146 if self.post_process and hasattr(self.storage, "post_process"):
147 processor = self.storage.post_process(found_files, dry_run=self.dry_run)
148 for original_path, processed_path, processed in processor:
149 if isinstance(processed, Exception):
150 self.stderr.write("Post-processing '%s' failed!" % original_path)
151 # Add a blank line before the traceback, otherwise it's
152 # too easy to miss the relevant part of the error message.
153 self.stderr.write()
154 raise processed
155 if processed:
156 self.log(
157 "Post-processed '%s' as '%s'" % (original_path, processed_path),
158 level=2,
159 )
160 self.post_processed_files.append(original_path)
161 else:
162 self.log("Skipped post-processing '%s'" % original_path)
163
164 return {
165 "modified": self.copied_files + self.symlinked_files,
166 "unmodified": self.unmodified_files,
167 "post_processed": self.post_processed_files,
168 }
169
170 def handle(self, **options):
171 self.set_options(**options)
172 message = ["\n"]
173 if self.dry_run:
174 message.append(
175 "You have activated the --dry-run option so no files will be "
176 "modified.\n\n"
177 )
178
179 message.append(
180 "You have requested to collect static files at the destination\n"
181 "location as specified in your settings"
182 )
183
184 if self.is_local_storage() and self.storage.location:
185 destination_path = self.storage.location
186 message.append(":\n\n %s\n\n" % destination_path)
187 should_warn_user = self.storage.exists(destination_path) and any(
188 self.storage.listdir(destination_path)
189 )
190 else:
191 destination_path = None
192 message.append(".\n\n")
193 # Destination files existence not checked; play it safe and warn.
194 should_warn_user = True
195
196 if self.interactive and should_warn_user:
197 if self.clear:
198 message.append("This will DELETE ALL FILES in this location!\n")
199 else:
200 message.append("This will overwrite existing files!\n")
201
202 message.append(
203 "Are you sure you want to do this?\n\n"
204 "Type 'yes' to continue, or 'no' to cancel: "
205 )
206 if input("".join(message)) != "yes":
207 raise CommandError("Collecting static files cancelled.")
208
209 collected = self.collect()
210
211 if self.verbosity >= 1:
212 modified_count = len(collected["modified"])
213 unmodified_count = len(collected["unmodified"])
214 post_processed_count = len(collected["post_processed"])
215 return (
216 "\n%(modified_count)s %(identifier)s %(action)s"
217 "%(destination)s%(unmodified)s%(post_processed)s."
218 ) % {
219 "modified_count": modified_count,
220 "identifier": "static file" + ("" if modified_count == 1 else "s"),
221 "action": "symlinked" if self.symlink else "copied",
222 "destination": (
223 " to '%s'" % destination_path if destination_path else ""
224 ),
225 "unmodified": (
226 ", %s unmodified" % unmodified_count
227 if collected["unmodified"]
228 else ""
229 ),
230 "post_processed": (
231 collected["post_processed"]
232 and ", %s post-processed" % post_processed_count
233 or ""
234 ),
235 }
236
237 def log(self, msg, level=2):
238 """
239 Small log helper
240 """
241 if self.verbosity >= level:
242 self.stdout.write(msg)
243
244 def is_local_storage(self):
245 return isinstance(self.storage, FileSystemStorage)
246
247 def clear_dir(self, path):
248 """
249 Delete the given relative path using the destination storage backend.
250 """
251 if not self.storage.exists(path):
252 return
253
254 dirs, files = self.storage.listdir(path)
255 for f in files:
256 fpath = os.path.join(path, f)
257 if self.dry_run:
258 self.log("Pretending to delete '%s'" % fpath, level=1)
259 else:
260 self.log("Deleting '%s'" % fpath, level=1)
261 try:
262 full_path = self.storage.path(fpath)
263 except NotImplementedError:
264 self.storage.delete(fpath)
265 else:
266 if not os.path.exists(full_path) and os.path.lexists(full_path):
267 # Delete broken symlinks
268 os.unlink(full_path)
269 else:
270 self.storage.delete(fpath)
271 for d in dirs:
272 self.clear_dir(os.path.join(path, d))
273
274 def delete_file(self, path, prefixed_path, source_storage):
275 """
276 Check if the target file should be deleted if it already exists.
277 """
278 if self.storage.exists(prefixed_path):
279 try:
280 # When was the target file modified last time?
281 target_last_modified = self.storage.get_modified_time(prefixed_path)
282 except (OSError, NotImplementedError):
283 # The storage doesn't support get_modified_time() or failed
284 pass
285 else:
286 try:
287 # When was the source file modified last time?
288 source_last_modified = source_storage.get_modified_time(path)
289 except (OSError, NotImplementedError):
290 pass
291 else:
292 # The full path of the target file
293 if self.local:
294 full_path = self.storage.path(prefixed_path)
295 # If it's --link mode and the path isn't a link (i.e.
296 # the previous collectstatic wasn't with --link) or if
297 # it's non-link mode and the path is a link (i.e. the
298 # previous collectstatic was with --link), the old
299 # links/files must be deleted so it's not safe to skip
300 # unmodified files.
301 can_skip_unmodified_files = not (
302 self.symlink ^ os.path.islink(full_path)
303 )
304 else:
305 # In remote storages, skipping is only based on the
306 # modified times since symlinks aren't relevant.
307 can_skip_unmodified_files = True
308 # Avoid sub-second precision (see #14665, #19540)
309 file_is_unmodified = target_last_modified.replace(
310 microsecond=0
311 ) >= source_last_modified.replace(microsecond=0)
312 if file_is_unmodified and can_skip_unmodified_files:
313 if prefixed_path not in self.unmodified_files:
314 self.unmodified_files.append(prefixed_path)
315 self.log("Skipping '%s' (not modified)" % path)
316 return False
317 # Then delete the existing file if really needed
318 if self.dry_run:
319 self.log("Pretending to delete '%s'" % path)
320 else:
321 self.log("Deleting '%s'" % path)
322 self.storage.delete(prefixed_path)
323 return True
324
325 def link_file(self, path, prefixed_path, source_storage):
326 """
327 Attempt to link ``path``
328 """
329 # Skip this file if it was already copied earlier
330 if prefixed_path in self.symlinked_files:
331 return self.log("Skipping '%s' (already linked earlier)" % path)
332 # Delete the target file if needed or break
333 if not self.delete_file(path, prefixed_path, source_storage):
334 return
335 # The full path of the source file
336 source_path = source_storage.path(path)
337 # Finally link the file
338 if self.dry_run:
339 self.log("Pretending to link '%s'" % source_path, level=1)
340 else:
341 self.log("Linking '%s'" % source_path, level=2)
342 full_path = self.storage.path(prefixed_path)
343 os.makedirs(os.path.dirname(full_path), exist_ok=True)
344 try:
345 if os.path.lexists(full_path):
346 os.unlink(full_path)
347 os.symlink(source_path, full_path)
348 except NotImplementedError:
349 import platform
350
351 raise CommandError(
352 "Symlinking is not supported in this "
353 "platform (%s)." % platform.platform()
354 )
355 except OSError as e:
356 raise CommandError(e)
357 if prefixed_path not in self.symlinked_files:
358 self.symlinked_files.append(prefixed_path)
359
360 def copy_file(self, path, prefixed_path, source_storage):
361 """
362 Attempt to copy ``path`` with storage
363 """
364 # Skip this file if it was already copied earlier
365 if prefixed_path in self.copied_files:
366 return self.log("Skipping '%s' (already copied earlier)" % path)
367 # Delete the target file if needed or break
368 if not self.delete_file(path, prefixed_path, source_storage):
369 return
370 # The full path of the source file
371 source_path = source_storage.path(path)
372 # Finally start copying
373 if self.dry_run:
374 self.log("Pretending to copy '%s'" % source_path, level=1)
375 else:
376 self.log("Copying '%s'" % source_path, level=2)
377 with source_storage.open(path) as source_file:
378 self.storage.save(prefixed_path, source_file)
379 self.copied_files.append(prefixed_path)