Debian Codegraph @

django/contrib/staticfiles/storage.py

raw
1 import json
2 import os
3 import posixpath
4 import re
5 from hashlib import md5
6 from urllib.parse import unquote, urldefrag, urlsplit, urlunsplit
7
8 from django.conf import STATICFILES_STORAGE_ALIAS, settings
9 from django.contrib.staticfiles.utils import check_settings, matches_patterns
10 from django.core.exceptions import ImproperlyConfigured
11 from django.core.files.base import ContentFile
12 from django.core.files.storage import FileSystemStorage, storages
13 from django.utils.functional import LazyObject
14
15
16 class StaticFilesStorage(FileSystemStorage):
17 """
18 Standard file system storage for static files.
19
20 The defaults for ``location`` and ``base_url`` are
21 ``STATIC_ROOT`` and ``STATIC_URL``.
22 """
23
24 def __init__(self, location=None, base_url=None, *args, **kwargs):
25 if location is None:
26 location = settings.STATIC_ROOT
27 if base_url is None:
28 base_url = settings.STATIC_URL
29 check_settings(base_url)
30 super().__init__(location, base_url, *args, **kwargs)
31 # FileSystemStorage fallbacks to MEDIA_ROOT when location
32 # is empty, so we restore the empty value.
33 if not location:
34 self.base_location = None
35 self.location = None
36
37 def path(self, name):
38 if not self.location:
39 raise ImproperlyConfigured(
40 "You're using the staticfiles app "
41 "without having set the STATIC_ROOT "
42 "setting to a filesystem path."
43 )
44 return super().path(name)
45
46
47 class HashedFilesMixin:
48 default_template = """url("%(url)s")"""
49 max_post_process_passes = 5
50 support_js_module_import_aggregation = False
51 _js_module_import_aggregation_patterns = (
52 "*.js",
53 (
54 (
55 (
56 r"""(?P<matched>import"""
57 r"""(?s:(?P<import>[\s\{].*?|\*\s*as\s*\w+))"""
58 r"""\s*from\s*['"](?P<url>[./].*?)["']\s*;)"""
59 ),
60 """import%(import)s from "%(url)s";""",
61 ),
62 (
63 (
64 r"""(?P<matched>export(?s:(?P<exports>[\s\{].*?))"""
65 r"""\s*from\s*["'](?P<url>[./].*?)["']\s*;)"""
66 ),
67 """export%(exports)s from "%(url)s";""",
68 ),
69 (
70 r"""(?P<matched>import\s*['"](?P<url>[./].*?)["']\s*;)""",
71 """import"%(url)s";""",
72 ),
73 (
74 r"""(?P<matched>import\(["'](?P<url>.*?)["']\))""",
75 """import("%(url)s")""",
76 ),
77 ),
78 )
79 patterns = (
80 (
81 "*.css",
82 (
83 r"""(?P<matched>url\(['"]{0,1}\s*(?P<url>.*?)["']{0,1}\))""",
84 (
85 r"""(?P<matched>@import\s*["']\s*(?P<url>.*?)["'])""",
86 """@import url("%(url)s")""",
87 ),
88 (
89 (
90 r"(?m)^(?P<matched>/\*#[ \t]"
91 r"(?-i:sourceMappingURL)=(?P<url>.*)[ \t]*\*/)$"
92 ),
93 "/*# sourceMappingURL=%(url)s */",
94 ),
95 ),
96 ),
97 (
98 "*.js",
99 (
100 (
101 r"(?m)^(?P<matched>//# (?-i:sourceMappingURL)=(?P<url>.*))$",
102 "//# sourceMappingURL=%(url)s",
103 ),
104 ),
105 ),
106 )
107 keep_intermediate_files = True
108
109 def __init__(self, *args, **kwargs):
110 if self.support_js_module_import_aggregation:
111 self.patterns += (self._js_module_import_aggregation_patterns,)
112 super().__init__(*args, **kwargs)
113 self._patterns = {}
114 self.hashed_files = {}
115 for extension, patterns in self.patterns:
116 for pattern in patterns:
117 if isinstance(pattern, (tuple, list)):
118 pattern, template = pattern
119 else:
120 template = self.default_template
121 compiled = re.compile(pattern, re.IGNORECASE)
122 self._patterns.setdefault(extension, []).append((compiled, template))
123
124 def file_hash(self, name, content=None):
125 """
126 Return a hash of the file with the given name and optional content.
127 """
128 if content is None:
129 return None
130 hasher = md5(usedforsecurity=False)
131 for chunk in content.chunks():
132 hasher.update(chunk)
133 return hasher.hexdigest()[:12]
134
135 def hashed_name(self, name, content=None, filename=None):
136 # `filename` is the name of file to hash if `content` isn't given.
137 # `name` is the base name to construct the new hashed filename from.
138 parsed_name = urlsplit(unquote(name))
139 clean_name = parsed_name.path.strip()
140 filename = (filename and urlsplit(unquote(filename)).path.strip()) or clean_name
141 opened = content is None
142 if opened:
143 if not self.exists(filename):
144 raise ValueError(
145 "The file '%s' could not be found with %r." % (filename, self)
146 )
147 try:
148 content = self.open(filename)
149 except OSError:
150 # Handle directory paths and fragments
151 return name
152 try:
153 file_hash = self.file_hash(clean_name, content)
154 finally:
155 if opened:
156 content.close()
157 path, filename = os.path.split(clean_name)
158 root, ext = os.path.splitext(filename)
159 file_hash = (".%s" % file_hash) if file_hash else ""
160 hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
161 unparsed_name = list(parsed_name)
162 unparsed_name[2] = hashed_name
163 # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
164 # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
165 if "?#" in name and not unparsed_name[3]:
166 unparsed_name[2] += "?"
167 return urlunsplit(unparsed_name)
168
169 def _url(self, hashed_name_func, name, force=False, hashed_files=None):
170 """
171 Return the non-hashed URL in DEBUG mode.
172 """
173 if settings.DEBUG and not force:
174 hashed_name, fragment = name, ""
175 else:
176 clean_name, fragment = urldefrag(name)
177 if urlsplit(clean_name).path.endswith("/"): # don't hash paths
178 hashed_name = name
179 else:
180 args = (clean_name,)
181 if hashed_files is not None:
182 args += (hashed_files,)
183 hashed_name = hashed_name_func(*args)
184
185 final_url = super().url(hashed_name)
186
187 # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
188 # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
189 query_fragment = "?#" in name # [sic!]
190 if fragment or query_fragment:
191 urlparts = list(urlsplit(final_url))
192 if fragment and not urlparts[4]:
193 urlparts[4] = fragment
194 if query_fragment and not urlparts[3]:
195 urlparts[2] += "?"
196 final_url = urlunsplit(urlparts)
197
198 return unquote(final_url)
199
200 def url(self, name, force=False):
201 """
202 Return the non-hashed URL in DEBUG mode.
203 """
204 return self._url(self.stored_name, name, force)
205
206 def url_converter(self, name, hashed_files, template=None):
207 """
208 Return the custom URL converter for the given file name.
209 """
210 if template is None:
211 template = self.default_template
212
213 def converter(matchobj):
214 """
215 Convert the matched URL to a normalized and hashed URL.
216
217 This requires figuring out which files the matched URL resolves
218 to and calling the url() method of the storage.
219 """
220 matches = matchobj.groupdict()
221 matched = matches["matched"]
222 url = matches["url"]
223
224 # Ignore absolute/protocol-relative and data-uri URLs.
225 if re.match(r"^[a-z]+:", url) or url.startswith("//"):
226 return matched
227
228 # Ignore absolute URLs that don't point to a static file (dynamic
229 # CSS / JS?). Note that STATIC_URL cannot be empty.
230 if url.startswith("/") and not url.startswith(settings.STATIC_URL):
231 return matched
232
233 # Strip off the fragment so a path-like fragment won't interfere.
234 url_path, fragment = urldefrag(url)
235
236 # Ignore URLs without a path
237 if not url_path:
238 return matched
239
240 if url_path.startswith("/"):
241 # Otherwise the condition above would have returned prematurely.
242 assert url_path.startswith(settings.STATIC_URL)
243 target_name = url_path.removeprefix(settings.STATIC_URL)
244 else:
245 # We're using the posixpath module to mix paths and URLs conveniently.
246 source_name = name if os.sep == "/" else name.replace(os.sep, "/")
247 target_name = posixpath.join(posixpath.dirname(source_name), url_path)
248
249 # Determine the hashed name of the target file with the storage backend.
250 hashed_url = self._url(
251 self._stored_name,
252 unquote(target_name),
253 force=True,
254 hashed_files=hashed_files,
255 )
256
257 transformed_url = "/".join(
258 url_path.split("/")[:-1] + hashed_url.split("/")[-1:]
259 )
260
261 # Restore the fragment that was stripped off earlier.
262 if fragment:
263 transformed_url += ("?#" if "?#" in url else "#") + fragment
264
265 # Return the hashed version to the file
266 matches["url"] = unquote(transformed_url)
267 return template % matches
268
269 return converter
270
271 def post_process(self, paths, dry_run=False, **options):
272 """
273 Post process the given dictionary of files (called from collectstatic).
274
275 Processing is actually two separate operations:
276
277 1. renaming files to include a hash of their content for cache-busting,
278 and copying those files to the target storage.
279 2. adjusting files which contain references to other files so they
280 refer to the cache-busting filenames.
281
282 If either of these are performed on a file, then that file is considered
283 post-processed.
284 """
285 # don't even dare to process the files if we're in dry run mode
286 if dry_run:
287 return
288
289 # where to store the new paths
290 hashed_files = {}
291
292 # build a list of adjustable files
293 adjustable_paths = [
294 path for path in paths if matches_patterns(path, self._patterns)
295 ]
296
297 # Adjustable files to yield at end, keyed by the original path.
298 processed_adjustable_paths = {}
299
300 # Do a single pass first. Post-process all files once, yielding not
301 # adjustable files and exceptions, and collecting adjustable files.
302 for name, hashed_name, processed, _ in self._post_process(
303 paths, adjustable_paths, hashed_files
304 ):
305 if name not in adjustable_paths or isinstance(processed, Exception):
306 yield name, hashed_name, processed
307 else:
308 processed_adjustable_paths[name] = (name, hashed_name, processed)
309
310 paths = {path: paths[path] for path in adjustable_paths}
311 unresolved_paths = []
312 for i in range(self.max_post_process_passes):
313 unresolved_paths = []
314 for name, hashed_name, processed, subst in self._post_process(
315 paths, adjustable_paths, hashed_files
316 ):
317 # Overwrite since hashed_name may be newer.
318 processed_adjustable_paths[name] = (name, hashed_name, processed)
319 if subst:
320 unresolved_paths.append(name)
321
322 if not unresolved_paths:
323 break
324
325 if unresolved_paths:
326 problem_paths = ", ".join(sorted(unresolved_paths))
327 yield problem_paths, None, RuntimeError("Max post-process passes exceeded.")
328
329 # Store the processed paths
330 self.hashed_files.update(hashed_files)
331
332 # Yield adjustable files with final, hashed name.
333 yield from processed_adjustable_paths.values()
334
335 def _post_process(self, paths, adjustable_paths, hashed_files):
336 # Sort the files by directory level
337 def path_level(name):
338 return len(name.split(os.sep))
339
340 for name in sorted(paths, key=path_level, reverse=True):
341 substitutions = True
342 # use the original, local file, not the copied-but-unprocessed
343 # file, which might be somewhere far away, like S3
344 storage, path = paths[name]
345 with storage.open(path) as original_file:
346 cleaned_name = self.clean_name(name)
347 hash_key = self.hash_key(cleaned_name)
348
349 # generate the hash with the original content, even for
350 # adjustable files.
351 if hash_key not in hashed_files:
352 hashed_name = self.hashed_name(name, original_file)
353 else:
354 hashed_name = hashed_files[hash_key]
355
356 # then get the original's file content..
357 if hasattr(original_file, "seek"):
358 original_file.seek(0)
359
360 hashed_file_exists = self.exists(hashed_name)
361 processed = False
362
363 # ..to apply each replacement pattern to the content
364 if name in adjustable_paths:
365 old_hashed_name = hashed_name
366 try:
367 content = original_file.read().decode("utf-8")
368 except UnicodeDecodeError as exc:
369 yield name, None, exc, False
370 for extension, patterns in self._patterns.items():
371 if matches_patterns(path, (extension,)):
372 for pattern, template in patterns:
373 converter = self.url_converter(
374 name, hashed_files, template
375 )
376 try:
377 content = pattern.sub(converter, content)
378 except ValueError as exc:
379 yield name, None, exc, False
380 if hashed_file_exists:
381 self.delete(hashed_name)
382 # then save the processed result
383 content_file = ContentFile(content.encode())
384 if self.keep_intermediate_files:
385 # Save intermediate file for reference
386 self._save(hashed_name, content_file)
387 hashed_name = self.hashed_name(name, content_file)
388
389 if self.exists(hashed_name):
390 self.delete(hashed_name)
391
392 saved_name = self._save(hashed_name, content_file)
393 hashed_name = self.clean_name(saved_name)
394 # If the file hash stayed the same, this file didn't change
395 if old_hashed_name == hashed_name:
396 substitutions = False
397 processed = True
398
399 if not processed:
400 # or handle the case in which neither processing nor
401 # a change to the original file happened
402 if not hashed_file_exists:
403 processed = True
404 saved_name = self._save(hashed_name, original_file)
405 hashed_name = self.clean_name(saved_name)
406
407 # and then set the cache accordingly
408 hashed_files[hash_key] = hashed_name
409
410 yield name, hashed_name, processed, substitutions
411
412 def clean_name(self, name):
413 return name.replace("\\", "/")
414
415 def hash_key(self, name):
416 return name
417
418 def _stored_name(self, name, hashed_files):
419 # Normalize the path to avoid multiple names for the same file like
420 # ../foo/bar.css and ../foo/../foo/bar.css which normalize to the same
421 # path.
422 name = posixpath.normpath(name)
423 cleaned_name = self.clean_name(name)
424 hash_key = self.hash_key(cleaned_name)
425 cache_name = hashed_files.get(hash_key)
426 if cache_name is None:
427 cache_name = self.clean_name(self.hashed_name(name))
428 return cache_name
429
430 def stored_name(self, name):
431 cleaned_name = self.clean_name(name)
432 hash_key = self.hash_key(cleaned_name)
433 cache_name = self.hashed_files.get(hash_key)
434 if cache_name:
435 return cache_name
436 # No cached name found, recalculate it from the files.
437 intermediate_name = name
438 for i in range(self.max_post_process_passes + 1):
439 cache_name = self.clean_name(
440 self.hashed_name(name, content=None, filename=intermediate_name)
441 )
442 if intermediate_name == cache_name:
443 # Store the hashed name if there was a miss.
444 self.hashed_files[hash_key] = cache_name
445 return cache_name
446 else:
447 # Move on to the next intermediate file.
448 intermediate_name = cache_name
449 # If the cache name can't be determined after the max number of passes,
450 # the intermediate files on disk may be corrupt; avoid an infinite loop.
451 raise ValueError("The name '%s' could not be hashed with %r." % (name, self))
452
453
454 class ManifestFilesMixin(HashedFilesMixin):
455 manifest_version = "1.1" # the manifest format standard
456 manifest_name = "staticfiles.json"
457 manifest_strict = True
458 keep_intermediate_files = False
459
460 def __init__(self, *args, manifest_storage=None, **kwargs):
461 super().__init__(*args, **kwargs)
462 if manifest_storage is None:
463 manifest_storage = self
464 self.manifest_storage = manifest_storage
465 self.hashed_files, self.manifest_hash = self.load_manifest()
466
467 def read_manifest(self):
468 try:
469 with self.manifest_storage.open(self.manifest_name) as manifest:
470 return manifest.read().decode()
471 except FileNotFoundError:
472 return None
473
474 def load_manifest(self):
475 content = self.read_manifest()
476 if content is None:
477 return {}, ""
478 try:
479 stored = json.loads(content)
480 except json.JSONDecodeError:
481 pass
482 else:
483 version = stored.get("version")
484 if version in ("1.0", "1.1"):
485 return stored.get("paths", {}), stored.get("hash", "")
486 raise ValueError(
487 "Couldn't load manifest '%s' (version %s)"
488 % (self.manifest_name, self.manifest_version)
489 )
490
491 def post_process(self, *args, **kwargs):
492 self.hashed_files = {}
493 yield from super().post_process(*args, **kwargs)
494 if not kwargs.get("dry_run"):
495 self.save_manifest()
496
497 def save_manifest(self):
498 self.manifest_hash = self.file_hash(
499 None, ContentFile(json.dumps(sorted(self.hashed_files.items())).encode())
500 )
501 payload = {
502 "paths": self.hashed_files,
503 "version": self.manifest_version,
504 "hash": self.manifest_hash,
505 }
506 if self.manifest_storage.exists(self.manifest_name):
507 self.manifest_storage.delete(self.manifest_name)
508 contents = json.dumps(payload).encode()
509 self.manifest_storage._save(self.manifest_name, ContentFile(contents))
510
511 def stored_name(self, name):
512 parsed_name = urlsplit(unquote(name))
513 clean_name = parsed_name.path.strip()
514 hash_key = self.hash_key(clean_name)
515 cache_name = self.hashed_files.get(hash_key)
516 if cache_name is None:
517 if self.manifest_strict:
518 raise ValueError(
519 "Missing staticfiles manifest entry for '%s'" % clean_name
520 )
521 cache_name = self.clean_name(self.hashed_name(name))
522 unparsed_name = list(parsed_name)
523 unparsed_name[2] = cache_name
524 # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
525 # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
526 if "?#" in name and not unparsed_name[3]:
527 unparsed_name[2] += "?"
528 return urlunsplit(unparsed_name)
529
530
531 class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):
532 """
533 A static file system storage backend which also saves
534 hashed copies of the files it saves.
535 """
536
537 pass
538
539
540 class ConfiguredStorage(LazyObject):
541 def _setup(self):
542 self._wrapped = storages[STATICFILES_STORAGE_ALIAS]
543
544
545 staticfiles_storage = ConfiguredStorage()