Debian Codegraph @

hugolib/filesystems/basefs.go

raw
1 // Copyright 2024 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 // Package filesystems provides the fine grained file systems used by Hugo. These
15 // are typically virtual filesystems that are composites of project and theme content.
16 package filesystems
17
18 import (
19 "context"
20 "fmt"
21 "io"
22 "os"
23 "path/filepath"
24 "strings"
25 "sync"
26
27 "github.com/bep/overlayfs"
28 "github.com/gohugoio/hugo/config"
29 "github.com/gohugoio/hugo/htesting"
30 "github.com/gohugoio/hugo/langs"
31
32 "github.com/gohugoio/hugo/common/herrors"
33 "github.com/gohugoio/hugo/common/hugo"
34 "github.com/gohugoio/hugo/common/loggers"
35
36 "github.com/rogpeppe/go-internal/lockedfile"
37
38 "github.com/gohugoio/hugo/hugofs/files"
39 "github.com/gohugoio/hugo/hugofs/hglob"
40
41 "github.com/gohugoio/hugo/modules"
42
43 hpaths "github.com/gohugoio/hugo/common/paths"
44 "github.com/gohugoio/hugo/hugofs"
45 "github.com/gohugoio/hugo/hugolib/paths"
46 "github.com/gohugoio/hugo/hugolib/sitesmatrix"
47 "github.com/spf13/afero"
48 )
49
50 const (
51 // Used to control concurrency between multiple Hugo instances, e.g.
52 // a running server and building new content with 'hugo new'.
53 // It's placed in the project root.
54 lockFileBuild = ".hugo_build.lock"
55 )
56
57 var filePathSeparator = string(filepath.Separator)
58
59 // BaseFs contains the core base filesystems used by Hugo. The name "base" is used
60 // to underline that even if they can be composites, they all have a base path set to a specific
61 // resource folder, e.g "/my-project/content". So, no absolute filenames needed.
62 type BaseFs struct {
63 // SourceFilesystems contains the different source file systems.
64 *SourceFilesystems
65
66 // The source filesystem (needs absolute filenames).
67 SourceFs afero.Fs
68
69 // The project source.
70 ProjectSourceFs afero.Fs
71
72 // The filesystem used to publish the rendered site.
73 // This usually maps to /my-project/public.
74 PublishFs afero.Fs
75
76 // The filesystem used for static files.
77 PublishFsStatic afero.Fs
78
79 // A read-only filesystem starting from the project workDir.
80 WorkDir afero.Fs
81
82 theBigFs *filesystemsCollector
83
84 workingDir string
85
86 // Locks.
87 buildMu Lockable // <project>/.hugo_build.lock
88 }
89
90 type Lockable interface {
91 Lock() (unlock func(), err error)
92 }
93
94 type fakeLockfileMutex struct {
95 mu sync.Mutex
96 }
97
98 func (f *fakeLockfileMutex) Lock() (func(), error) {
99 f.mu.Lock()
100 return func() { f.mu.Unlock() }, nil
101 }
102
103 // Tries to acquire a build lock.
104 func (b *BaseFs) LockBuild() (unlock func(), err error) {
105 return b.buildMu.Lock()
106 }
107
108 func (b *BaseFs) WatchFilenames() []string {
109 var filenames []string
110 sourceFs := b.SourceFs
111
112 for _, rfs := range b.RootFss {
113 for _, component := range files.ComponentFolders {
114 fis, err := rfs.Mounts(component)
115 if err != nil {
116 continue
117 }
118
119 for _, fim := range fis {
120 meta := fim.Meta()
121 if !meta.Watch {
122 continue
123 }
124
125 if !fim.IsDir() {
126 filenames = append(filenames, meta.Filename)
127 continue
128 }
129
130 w := hugofs.NewWalkway(hugofs.WalkwayConfig{
131 Fs: sourceFs,
132 Root: meta.Filename,
133 WalkFn: func(ctx context.Context, path string, fi hugofs.FileMetaInfo) error {
134 if !fi.IsDir() {
135 return nil
136 }
137 if fi.Name() == ".git" ||
138 fi.Name() == "node_modules" || fi.Name() == "bower_components" {
139 return filepath.SkipDir
140 }
141 filenames = append(filenames, fi.Meta().Filename)
142 return nil
143 },
144 })
145
146 w.Walk()
147 }
148
149 }
150 }
151
152 return filenames
153 }
154
155 func (b *BaseFs) mountsForComponent(component string) []hugofs.FileMetaInfo {
156 var result []hugofs.FileMetaInfo
157 for _, rfs := range b.RootFss {
158 dirs, err := rfs.Mounts(component)
159 if err == nil {
160 result = append(result, dirs...)
161 }
162 }
163 return result
164 }
165
166 // AbsProjectContentDir tries to construct a filename below the most
167 // relevant content directory.
168 func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
169 isAbs := filepath.IsAbs(filename)
170 for _, fi := range b.mountsForComponent(files.ComponentFolderContent) {
171 if !fi.IsDir() {
172 continue
173 }
174 meta := fi.Meta()
175 if !meta.IsProject {
176 continue
177 }
178
179 if isAbs {
180 if strings.HasPrefix(filename, meta.Filename) {
181 return strings.TrimPrefix(filename, meta.Filename+filePathSeparator), filename, nil
182 }
183 } else {
184 contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + filePathSeparator
185
186 if after, ok := strings.CutPrefix(filename, contentDir); ok {
187 relFilename := after
188 absFilename := filepath.Join(meta.Filename, relFilename)
189 return relFilename, absFilename, nil
190 }
191 }
192
193 }
194
195 if !isAbs {
196 // A filename on the form "posts/mypage.md", put it inside
197 // the first content folder, usually <workDir>/content.
198 // Pick the first project dir (which is probably the most important one).
199 for _, dir := range b.SourceFilesystems.Content.mounts() {
200 if !dir.IsDir() {
201 continue
202 }
203 meta := dir.Meta()
204 if meta.IsProject {
205 return filename, filepath.Join(meta.Filename, filename), nil
206 }
207 }
208 }
209
210 return "", "", fmt.Errorf("could not determine content directory for %q", filename)
211 }
212
213 // ResolveJSConfigFile resolves the JS-related config file to a absolute
214 // filename. One example of such would be postcss.config.js.
215 func (b *BaseFs) ResolveJSConfigFile(name string) string {
216 // First look in assets/_jsconfig
217 fi, err := b.Assets.Fs.Stat(filepath.Join(files.FolderJSConfig, name))
218 if err == nil {
219 return fi.(hugofs.FileMetaInfo).Meta().Filename
220 }
221 // Fall back to the work dir.
222 fi, err = b.Work.Stat(name)
223 if err == nil {
224 return fi.(hugofs.FileMetaInfo).Meta().Filename
225 }
226
227 return ""
228 }
229
230 // SourceFilesystems contains the different source file systems. These can be
231 // composite file systems (theme and project etc.), and they have all root
232 // set to the source type the provides: data, i18n, static, layouts.
233 type SourceFilesystems struct {
234 Content *SourceFilesystem
235 Data *SourceFilesystem
236 I18n *SourceFilesystem
237 Layouts *SourceFilesystem
238 Archetypes *SourceFilesystem
239 Assets *SourceFilesystem
240
241 AssetsWithDuplicatesPreserved *SourceFilesystem
242
243 RootFss []*hugofs.RootMappingFs
244
245 // Writable filesystem on top the project's resources directory,
246 // with any sub module's resource fs layered below.
247 ResourcesCache afero.Fs
248
249 // The work folder (may be a composite of project and theme components).
250 Work afero.Fs
251
252 // When in multihost we have one static filesystem per language. The sync
253 // static files is currently done outside of the Hugo build (where there is
254 // a concept of a site per language).
255 // When in non-multihost mode there will be one entry in this map with a blank key.
256 Static map[string]*SourceFilesystem
257
258 conf config.AllProvider
259 }
260
261 // A SourceFilesystem holds the filesystem for a given source type in Hugo (data,
262 // i18n, layouts, static) and additional metadata to be able to use that filesystem
263 // in server mode.
264 type SourceFilesystem struct {
265 // Name matches one in files.ComponentFolders
266 Name string
267
268 // This is a virtual composite filesystem. It expects path relative to a context.
269 Fs afero.Fs
270
271 // The source filesystem (usually the OS filesystem).
272 SourceFs afero.Fs
273
274 // When syncing a source folder to the target (e.g. /public), this may
275 // be set to publish into a subfolder. This is used for static syncing
276 // in multihost mode.
277 PublishFolder string
278 }
279
280 // StaticFs returns the static filesystem for the given language.
281 // This can be a composite filesystem.
282 func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
283 var staticFs afero.Fs = hugofs.NoOpFs
284
285 if fs, ok := s.Static[lang]; ok {
286 staticFs = fs.Fs
287 } else if fs, ok := s.Static[""]; ok {
288 staticFs = fs.Fs
289 }
290
291 return staticFs
292 }
293
294 // StatResource looks for a resource in these filesystems in order: static, assets and finally content.
295 // If found in any of them, it returns FileInfo and the relevant filesystem.
296 // Any non herrors.IsNotExist error will be returned.
297 // An herrors.IsNotExist error will be returned only if all filesystems return such an error.
298 // Note that if we only wanted to find the file, we could create a composite Afero fs,
299 // but we also need to know which filesystem root it lives in.
300 func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) {
301 for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} {
302 fs = fsToCheck
303 fi, err = fs.Stat(filename)
304 if err == nil || !herrors.IsNotExist(err) {
305 return
306 }
307 }
308 // Not found.
309 return
310 }
311
312 // IsStatic returns true if the given filename is a member of one of the static
313 // filesystems.
314 func (s SourceFilesystems) IsStatic(filename string) bool {
315 for _, staticFs := range s.Static {
316 if staticFs.Contains(filename) {
317 return true
318 }
319 }
320 return false
321 }
322
323 // IsContent returns true if the given filename is a member of the content filesystem.
324 func (s SourceFilesystems) IsContent(filename string) bool {
325 return s.Content.Contains(filename)
326 }
327
328 // ResolvePaths resolves the given filename to a list of paths in the filesystems.
329 func (s *SourceFilesystems) ResolvePaths(filename string) []hugofs.ComponentPath {
330 var cpss []hugofs.ComponentPath
331 for _, rfs := range s.RootFss {
332 cps, err := rfs.ReverseLookup(filename)
333 if err != nil {
334 panic(err)
335 }
336 cpss = append(cpss, cps...)
337 }
338 return cpss
339 }
340
341 // MakeStaticPathRelative makes an absolute static filename into a relative one.
342 // It will return an empty string if the filename is not a member of a static filesystem.
343 func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
344 for _, staticFs := range s.Static {
345 rel, _ := staticFs.MakePathRelative(filename, true)
346 if rel != "" {
347 return rel
348 }
349 }
350 return ""
351 }
352
353 // MakePathRelative creates a relative path from the given filename.
354 func (d *SourceFilesystem) MakePathRelative(filename string, checkExists bool) (string, bool) {
355 cps, err := d.ReverseLookup(filename, checkExists)
356 if err != nil {
357 panic(err)
358 }
359 if len(cps) == 0 {
360 return "", false
361 }
362
363 return filepath.FromSlash(cps[0].Path), true
364 }
365
366 // ReverseLookup returns the component paths for the given filename.
367 func (d *SourceFilesystem) ReverseLookup(filename string, checkExists bool) ([]hugofs.ComponentPath, error) {
368 var cps []hugofs.ComponentPath
369 hugofs.WalkFilesystems(d.Fs, func(fs afero.Fs) bool {
370 if rfs, ok := fs.(hugofs.ReverseLookupProvder); ok {
371 if c, err := rfs.ReverseLookupComponent(d.Name, filename); err == nil {
372 if checkExists {
373 n := 0
374 for _, cp := range c {
375 if _, err := d.Fs.Stat(filepath.FromSlash(cp.Path)); err == nil {
376 c[n] = cp
377 n++
378 }
379 }
380 c = c[:n]
381 }
382 cps = append(cps, c...)
383 }
384 }
385 return false
386 })
387 return cps, nil
388 }
389
390 func (d *SourceFilesystem) mounts() []hugofs.FileMetaInfo {
391 var m []hugofs.FileMetaInfo
392 hugofs.WalkFilesystems(d.Fs, func(fs afero.Fs) bool {
393 if rfs, ok := fs.(*hugofs.RootMappingFs); ok {
394 mounts, err := rfs.Mounts(d.Name)
395 if err == nil {
396 m = append(m, mounts...)
397 }
398 }
399 return false
400 })
401
402 // Filter out any mounts not belonging to this filesystem.
403 // TODO(bep) I think this is superfluous.
404 n := 0
405 for _, mm := range m {
406 if mm.Meta().Component == d.Name {
407 m[n] = mm
408 n++
409 }
410 }
411 m = m[:n]
412
413 return m
414 }
415
416 func (d *SourceFilesystem) RealFilename(rel string) string {
417 fi, err := d.Fs.Stat(rel)
418 if err != nil {
419 return rel
420 }
421 if realfi, ok := fi.(hugofs.FileMetaInfo); ok {
422 return realfi.Meta().Filename
423 }
424
425 return rel
426 }
427
428 // Contains returns whether the given filename is a member of the current filesystem.
429 func (d *SourceFilesystem) Contains(filename string) bool {
430 for _, dir := range d.mounts() {
431 if !dir.IsDir() {
432 continue
433 }
434 if strings.HasPrefix(filename, dir.Meta().Filename) {
435 return true
436 }
437 }
438 return false
439 }
440
441 // RealDirs gets a list of absolute paths to directories starting from the given
442 // path.
443 func (d *SourceFilesystem) RealDirs(from string) []string {
444 var dirnames []string
445 for _, m := range d.mounts() {
446 if !m.IsDir() {
447 continue
448 }
449 dirname := filepath.Join(m.Meta().Filename, from)
450 if _, err := d.SourceFs.Stat(dirname); err == nil {
451 dirnames = append(dirnames, dirname)
452 }
453 }
454 return dirnames
455 }
456
457 // RealPaths returns the absolute paths of all mounts in this filesystem.
458 func (d *SourceFilesystem) RealPaths(from string) []string {
459 var paths []string
460 for _, m := range d.mounts() {
461 p := filepath.Join(m.Meta().Filename, from)
462 if _, err := d.SourceFs.Stat(p); err == nil {
463 paths = append(paths, p)
464 }
465 }
466 return paths
467 }
468
469 // WithBaseFs allows reuse of some potentially expensive to create parts that remain
470 // the same across sites/languages.
471 func WithBaseFs(b *BaseFs) func(*BaseFs) error {
472 return func(bb *BaseFs) error {
473 bb.theBigFs = b.theBigFs
474 bb.SourceFilesystems = b.SourceFilesystems
475 return nil
476 }
477 }
478
479 // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
480 func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) {
481 fs := p.Fs
482 if logger == nil {
483 logger = loggers.NewDefault()
484 }
485
486 publishFs := hugofs.NewBaseFileDecorator(fs.PublishDir)
487 projectSourceFs := hugofs.NewBaseFileDecorator(hugofs.NewBasePathFs(fs.Source, p.Cfg.BaseConfig().WorkingDir))
488 sourceFs := hugofs.NewBaseFileDecorator(fs.Source)
489 publishFsStatic := fs.PublishDirStatic
490
491 var buildMu Lockable
492 if p.Cfg.NoBuildLock() || htesting.IsTest {
493 buildMu = &fakeLockfileMutex{}
494 } else {
495 buildMu = lockedfile.MutexAt(filepath.Join(p.Cfg.BaseConfig().WorkingDir, lockFileBuild))
496 }
497
498 b := &BaseFs{
499 SourceFs: sourceFs,
500 ProjectSourceFs: projectSourceFs,
501 WorkDir: fs.WorkingDirReadOnly,
502 PublishFs: publishFs,
503 PublishFsStatic: publishFsStatic,
504 workingDir: p.Cfg.BaseConfig().WorkingDir,
505 buildMu: buildMu,
506 }
507
508 for _, opt := range options {
509 if err := opt(b); err != nil {
510 return nil, err
511 }
512 }
513
514 if b.theBigFs != nil && b.SourceFilesystems != nil {
515 return b, nil
516 }
517
518 builder := newSourceFilesystemsBuilder(p, logger, b)
519 sourceFilesystems, err := builder.Build()
520 if err != nil {
521 return nil, fmt.Errorf("build filesystems: %w", err)
522 }
523
524 b.SourceFilesystems = sourceFilesystems
525 b.theBigFs = builder.theBigFs
526
527 return b, nil
528 }
529
530 type sourceFilesystemsBuilder struct {
531 logger loggers.Logger
532 p *paths.Paths
533 sourceFs afero.Fs
534 result *SourceFilesystems
535 theBigFs *filesystemsCollector
536 }
537
538 func newSourceFilesystemsBuilder(p *paths.Paths, logger loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder {
539 sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source)
540 return &sourceFilesystemsBuilder{
541 p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs,
542 result: &SourceFilesystems{
543 conf: p.Cfg,
544 },
545 }
546 }
547
548 func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs) *SourceFilesystem {
549 return &SourceFilesystem{
550 Name: name,
551 Fs: fs,
552 SourceFs: b.sourceFs,
553 }
554 }
555
556 func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
557 if b.theBigFs == nil {
558 theBigFs, err := b.createMainOverlayFs(b.p)
559 if err != nil {
560 return nil, fmt.Errorf("create main fs: %w", err)
561 }
562
563 b.theBigFs = theBigFs
564 }
565
566 createView := func(componentID string, overlayFs *overlayfs.OverlayFs) *SourceFilesystem {
567 if b.theBigFs == nil || b.theBigFs.overlayMounts == nil {
568 return b.newSourceFilesystem(componentID, hugofs.NoOpFs)
569 }
570
571 fs := hugofs.NewComponentFs(
572 hugofs.ComponentFsOptions{
573 Fs: overlayFs,
574 Component: componentID,
575 Cfg: b.p.Cfg,
576 },
577 )
578
579 return b.newSourceFilesystem(componentID, fs)
580 }
581
582 b.result.Archetypes = createView(files.ComponentFolderArchetypes, b.theBigFs.overlayMounts)
583 b.result.Layouts = createView(files.ComponentFolderLayouts, b.theBigFs.overlayMounts)
584 b.result.Assets = createView(files.ComponentFolderAssets, b.theBigFs.overlayMounts)
585 b.result.ResourcesCache = b.theBigFs.overlayResources
586 b.result.RootFss = b.theBigFs.rootFss
587
588 // data and i18n needs a different merge strategy.
589 overlayMountsPreserveDupes := b.theBigFs.overlayMounts.WithDirsMerger(hugofs.AppendDirsMerger)
590 b.result.Data = createView(files.ComponentFolderData, overlayMountsPreserveDupes)
591 b.result.I18n = createView(files.ComponentFolderI18n, overlayMountsPreserveDupes)
592 b.result.AssetsWithDuplicatesPreserved = createView(files.ComponentFolderAssets, overlayMountsPreserveDupes)
593
594 contentFs := hugofs.NewComponentFs(
595 hugofs.ComponentFsOptions{
596 Fs: b.theBigFs.overlayMountsContent,
597 Component: files.ComponentFolderContent,
598 Cfg: b.p.Cfg,
599 },
600 )
601
602 b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs)
603 b.result.Work = hugofs.NewReadOnlyFs(b.theBigFs.overlayFull)
604
605 // Create static filesystem(s)
606 ms := make(map[string]*SourceFilesystem)
607 b.result.Static = ms
608
609 if b.theBigFs.staticPerLanguage != nil {
610 // Multihost mode
611 for k, v := range b.theBigFs.staticPerLanguage {
612 sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v)
613 sfs.PublishFolder = k
614 ms[k] = sfs
615 }
616 } else {
617 bfs := hugofs.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic)
618 ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs)
619 }
620
621 return b.result, nil
622 }
623
624 func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) {
625 var staticFsMap map[string]*overlayfs.OverlayFs
626 if b.p.Cfg.IsMultihost() {
627 languages := b.p.Cfg.Languages().(langs.Languages)
628 staticFsMap = make(map[string]*overlayfs.OverlayFs)
629 for _, l := range languages {
630 staticFsMap[l.Lang] = overlayfs.New(overlayfs.Options{})
631 }
632 }
633
634 collector := &filesystemsCollector{
635 sourceProject: b.sourceFs,
636 sourceModules: b.sourceFs,
637 staticPerLanguage: staticFsMap,
638
639 overlayMounts: overlayfs.New(overlayfs.Options{}),
640 overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.AppendDirsMerger}),
641 overlayMountsStatic: overlayfs.New(overlayfs.Options{}),
642 overlayFull: overlayfs.New(overlayfs.Options{}),
643 overlayResources: overlayfs.New(overlayfs.Options{FirstWritable: true}),
644 }
645
646 mods := p.AllModules()
647
648 mounts := make([]mountsDescriptor, len(mods))
649
650 for i := range mods {
651 mod := mods[i]
652 dir := mod.Dir()
653
654 isMainProject := mod.Owner() == nil
655 mounts[i] = mountsDescriptor{
656 Module: mod,
657 dir: dir,
658 isMainProject: isMainProject,
659 ordinal: i,
660 }
661
662 }
663
664 err := b.createOverlayFs(collector, mounts)
665
666 return collector, err
667 }
668
669 func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool {
670 return strings.HasPrefix(mnt.Target, files.ComponentFolderContent)
671 }
672
673 func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool {
674 return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic)
675 }
676
677 func (b *sourceFilesystemsBuilder) isLayoutsMount(mnt modules.Mount) bool {
678 return strings.HasPrefix(mnt.Target, files.ComponentFolderLayouts)
679 }
680
681 func (b *sourceFilesystemsBuilder) createOverlayFs(
682 collector *filesystemsCollector,
683 mounts []mountsDescriptor,
684 ) error {
685 if len(mounts) == 0 {
686 appendNopIfEmpty := func(ofs *overlayfs.OverlayFs) *overlayfs.OverlayFs {
687 if ofs.NumFilesystems() > 0 {
688 return ofs
689 }
690 return ofs.Append(hugofs.NoOpFs)
691 }
692 collector.overlayMounts = appendNopIfEmpty(collector.overlayMounts)
693 collector.overlayMountsContent = appendNopIfEmpty(collector.overlayMountsContent)
694 collector.overlayMountsStatic = appendNopIfEmpty(collector.overlayMountsStatic)
695 collector.overlayMountsFull = appendNopIfEmpty(collector.overlayMountsFull)
696 collector.overlayFull = appendNopIfEmpty(collector.overlayFull)
697 collector.overlayResources = appendNopIfEmpty(collector.overlayResources)
698
699 return nil
700 }
701
702 for _, md := range mounts {
703 var (
704 fromTo []*hugofs.RootMapping
705 fromToContent []*hugofs.RootMapping
706 fromToStatic []*hugofs.RootMapping
707 )
708
709 absPathify := func(path string) (string, string) {
710 if filepath.IsAbs(path) {
711 return "", path
712 }
713 return md.dir, hpaths.AbsPathify(md.dir, path)
714 }
715
716 for i, mount := range md.Mounts() {
717 // Add more weight to early mounts.
718 // When two mounts contain the same filename,
719 // the first entry wins.
720 mountWeight := (10 + md.ordinal) * (len(md.Mounts()) - i)
721
722 patterns, hasLegacyIncludes, hasLegacyExcludes := mount.FilesToFilter()
723 if hasLegacyIncludes {
724 hugo.Deprecate("module.mounts.includeFiles", "Replaced by the simpler 'files' setting, see https://gohugo.io/configuration/module/#files", "v0.153.0")
725 }
726 if hasLegacyExcludes {
727 hugo.Deprecate("module.mounts.excludeFiles", "Replaced by the simpler 'files' setting, see https://gohugo.io/configuration/module/#files", "v0.153.0")
728 }
729
730 inclusionFilter, err := hglob.NewFilenameFilterV2(patterns)
731 if err != nil {
732 return err
733 }
734
735 base, filename := absPathify(mount.Source)
736
737 var matrix *sitesmatrix.IntSets
738 v := mount.Sites
739
740 needsDefaultsIfNotset := b.isContentMount(mount)
741 needsDefaultsAndAllLanguagesIfNotSet := b.isStaticMount(mount)
742 needsAllIfNotSet := b.isLayoutsMount(mount)
743
744 intSetsCfg := sitesmatrix.IntSetsConfig{
745 ApplyDefaults: 0,
746 Globs: v.Matrix,
747 }
748
749 matrixBuilder := sitesmatrix.NewIntSetsBuilder(b.p.Cfg.ConfiguredDimensions())
750
751 if !v.Matrix.IsZero() {
752 matrixBuilder.WithConfig(intSetsCfg)
753 if !matrixBuilder.GlobFilterMisses.IsZero() {
754 continue
755 }
756
757 if needsDefaultsIfNotset {
758 matrixBuilder.WithDefaultsIfNotSet()
759 }
760 if needsAllIfNotSet {
761 matrixBuilder.WithAllIfNotSet()
762 }
763
764 matrix = matrixBuilder.Build()
765 } else if needsAllIfNotSet {
766 matrixBuilder.WithAllIfNotSet()
767 matrix = matrixBuilder.Build()
768 } else if needsDefaultsAndAllLanguagesIfNotSet {
769 matrixBuilder.WithDefaultsAndAllLanguagesIfNotSet()
770 matrix = matrixBuilder.Build()
771 } else if needsDefaultsIfNotset {
772 matrix = b.p.Cfg.DefaultContentsitesMatrix()
773 } else {
774 if needsDefaultsIfNotset {
775 matrixBuilder.WithDefaultsIfNotSet()
776 }
777 matrix = matrixBuilder.Build()
778 }
779
780 var complements *sitesmatrix.IntSets
781 if !v.Complements.IsZero() {
782 intSetsCfg.Globs = v.Complements
783 intSetsCfg.ApplyDefaults = 0
784 matrixBuilder = sitesmatrix.NewIntSetsBuilder(b.p.Cfg.ConfiguredDimensions()).WithConfig(intSetsCfg).WithAllIfNotSet()
785 complements = matrixBuilder.Build()
786 }
787
788 rm := &hugofs.RootMapping{
789 From: mount.Target,
790 To: filename,
791 ToBase: base,
792 Module: md.Module,
793 ModuleOrdinal: md.ordinal,
794 IsProject: md.isMainProject,
795 Meta: &hugofs.FileMeta{
796 Watch: !mount.DisableWatch && md.Watch(),
797 Weight: mountWeight,
798 InclusionFilter: inclusionFilter,
799 SitesMatrix: matrix,
800 SitesComplements: complements,
801 },
802 }
803
804 isContentMount := b.isContentMount(mount)
805
806 if isContentMount {
807 fromToContent = append(fromToContent, rm)
808 } else if b.isStaticMount(mount) {
809 fromToStatic = append(fromToStatic, rm)
810 } else {
811 fromTo = append(fromTo, rm)
812 }
813 }
814
815 modBase := collector.sourceProject
816 if !md.isMainProject {
817 modBase = collector.sourceModules
818 }
819
820 sourceStatic := modBase
821
822 rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
823 if err != nil {
824 return err
825 }
826 rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
827 if err != nil {
828 return err
829 }
830 rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)
831 if err != nil {
832 return err
833 }
834
835 // We need to keep the list of directories for watching.
836 collector.addRootFs(rmfs)
837 collector.addRootFs(rmfsContent)
838 collector.addRootFs(rmfsStatic)
839
840 if collector.staticPerLanguage != nil {
841 for i, l := range b.p.Cfg.Languages().(langs.Languages) {
842 lang := l.Lang
843
844 lfs := rmfsStatic.Filter(func(rm *hugofs.RootMapping) bool {
845 return rm.Meta.SitesMatrix.HasLanguage(i)
846 })
847 bfs := hugofs.NewBasePathFs(lfs, files.ComponentFolderStatic)
848 collector.staticPerLanguage[lang] = collector.staticPerLanguage[lang].Append(bfs)
849 }
850 }
851
852 getResourcesDir := func() string {
853 if md.isMainProject {
854 return b.p.AbsResourcesDir
855 }
856 _, filename := absPathify(files.FolderResources)
857 return filename
858 }
859
860 collector.overlayMounts = collector.overlayMounts.Append(rmfs)
861 collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
862 collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
863 collector.overlayFull = collector.overlayFull.Append(hugofs.NewBasePathFs(modBase, md.dir))
864 collector.overlayResources = collector.overlayResources.Append(hugofs.NewBasePathFs(modBase, getResourcesDir()))
865
866 }
867
868 return nil
869 }
870
871 //lint:ignore U1000 useful for debugging
872 func printFs(fs afero.Fs, path string, w io.Writer) {
873 if fs == nil {
874 return
875 }
876 afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
877 if err != nil {
878 return err
879 }
880 if info.IsDir() {
881 return nil
882 }
883 var filename string
884 if fim, ok := info.(hugofs.FileMetaInfo); ok {
885 filename = fim.Meta().Filename
886 }
887 fmt.Fprintf(w, " %q %q\n", path, filename)
888 return nil
889 })
890 }
891
892 type filesystemsCollector struct {
893 sourceProject afero.Fs // Source for project folders
894 sourceModules afero.Fs // Source for modules/themes
895
896 overlayMounts *overlayfs.OverlayFs
897 overlayMountsContent *overlayfs.OverlayFs
898 overlayMountsStatic *overlayfs.OverlayFs
899 overlayMountsFull *overlayfs.OverlayFs
900 overlayFull *overlayfs.OverlayFs
901 overlayResources *overlayfs.OverlayFs
902
903 rootFss []*hugofs.RootMappingFs
904
905 // Set if in multihost mode
906 staticPerLanguage map[string]*overlayfs.OverlayFs
907 }
908
909 func (c *filesystemsCollector) addRootFs(rfs *hugofs.RootMappingFs) {
910 c.rootFss = append(c.rootFss, rfs)
911 }
912
913 type mountsDescriptor struct {
914 modules.Module
915 dir string
916 isMainProject bool
917 ordinal int // zero based starting from the project.
918 }