Debian Codegraph @

buildkit/frontend/gateway/container/container.go

raw
1 package container
2
3 import (
4 "cmp"
5 "context"
6 "fmt"
7 "path/filepath"
8 "runtime"
9 "slices"
10 "strings"
11 "sync"
12 "syscall"
13
14 "github.com/moby/buildkit/session/secrets"
15 "github.com/moby/buildkit/util/bklog"
16 "github.com/moby/buildkit/util/system"
17
18 "github.com/moby/buildkit/cache"
19 "github.com/moby/buildkit/executor"
20 "github.com/moby/buildkit/frontend/gateway/client"
21 "github.com/moby/buildkit/session"
22 "github.com/moby/buildkit/snapshot"
23 "github.com/moby/buildkit/solver/llbsolver/mounts"
24 opspb "github.com/moby/buildkit/solver/pb"
25 "github.com/moby/buildkit/util/stack"
26 "github.com/moby/buildkit/worker"
27 "github.com/pkg/errors"
28 "golang.org/x/sync/errgroup"
29 )
30
31 type NewContainerRequest struct {
32 ContainerID string
33 NetMode opspb.NetMode
34 Hostname string
35 ExtraHosts []executor.HostIP
36 Mounts []Mount
37 Platform *opspb.Platform
38 Constraints *opspb.WorkerConstraints
39 }
40
41 // Mount used for the gateway.Container is nearly identical to the client.Mount
42 // except is has a RefProxy instead of Ref to allow for a common abstraction
43 // between gateway clients.
44 type Mount struct {
45 *opspb.Mount
46 WorkerRef *worker.WorkerRef
47 }
48
49 func NewContainer(ctx context.Context, cm cache.Manager, exec executor.Executor, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
50 ctx, cancel := context.WithCancelCause(ctx)
51 eg, ctx := errgroup.WithContext(ctx)
52 platform := &opspb.Platform{
53 OS: runtime.GOOS,
54 Architecture: runtime.GOARCH,
55 }
56 if req.Platform != nil {
57 platform = req.Platform
58 }
59 ctr := &gatewayContainer{
60 id: req.ContainerID,
61 netMode: req.NetMode,
62 hostname: req.Hostname,
63 extraHosts: req.ExtraHosts,
64 platform: platform,
65 executor: exec,
66 sm: sm,
67 group: g,
68 errGroup: eg,
69 ctx: ctx,
70 cancel: cancel,
71 }
72
73 var (
74 mnts []*opspb.Mount
75 refs []*worker.WorkerRef
76 )
77 for _, m := range req.Mounts {
78 mnts = append(mnts, m.Mount)
79 if m.WorkerRef != nil {
80 refs = append(refs, m.WorkerRef)
81 m.Input = int64(len(refs) - 1)
82 } else {
83 m.Input = int64(opspb.Empty)
84 }
85 }
86
87 name := fmt.Sprintf("container %s", req.ContainerID)
88 mm := mounts.NewMountManager(name, cm, sm)
89 p, err := PrepareMounts(ctx, mm, cm, g, "", mnts, refs, func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
90 if m.Input != int64(opspb.Empty) {
91 cm = refs[m.Input].Worker.CacheManager()
92 }
93 return cm.New(ctx, ref, g)
94 }, platform.OS)
95 if err != nil {
96 for i := len(p.Actives) - 1; i >= 0; i-- { // call in LIFO order
97 p.Actives[i].Ref.Release(context.TODO())
98 }
99 for _, o := range p.OutputRefs {
100 o.Ref.Release(context.TODO())
101 }
102 return nil, err
103 }
104 ctr.rootFS = p.Root
105 ctr.mounts = p.Mounts
106
107 for _, o := range p.OutputRefs {
108 ctr.cleanup = append(ctr.cleanup, func() error {
109 return o.Ref.Release(context.TODO())
110 })
111 }
112 for _, active := range p.Actives {
113 ctr.cleanup = append(ctr.cleanup, func() error {
114 return active.Ref.Release(context.TODO())
115 })
116 }
117
118 return ctr, nil
119 }
120
121 type PreparedMounts struct {
122 Root executor.Mount
123 ReadonlyRootFS bool
124 Mounts []executor.Mount
125 OutputRefs []MountRef
126 Actives []MountMutableRef
127 }
128
129 type MountRef struct {
130 Ref cache.Ref
131 MountIndex int
132 }
133
134 type MountMutableRef struct {
135 Ref cache.MutableRef
136 MountIndex int
137 NoCommit bool
138 }
139
140 type MakeMutable func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error)
141
142 func PrepareMounts(ctx context.Context, mm *mounts.MountManager, cm cache.Manager, g session.Group, cwd string, mnts []*opspb.Mount, refs []*worker.WorkerRef, makeMutable MakeMutable, platform string) (p PreparedMounts, err error) {
143 // loop over all mounts, fill in mounts, root and outputs
144 for i, m := range mnts {
145 var (
146 mountable cache.Mountable
147 ref cache.ImmutableRef
148 )
149
150 if m.Dest == opspb.RootMount && m.MountType != opspb.MountType_BIND {
151 return p, errors.Errorf("invalid mount type %s for %s", m.MountType.String(), m.Dest)
152 }
153
154 // if mount is based on input validate and load it
155 if m.Input != int64(opspb.Empty) {
156 if int(m.Input) >= len(refs) {
157 return p, errors.Errorf("missing input %d", m.Input)
158 }
159 ref = refs[int(m.Input)].ImmutableRef
160 mountable = ref
161 }
162
163 switch m.MountType {
164 case opspb.MountType_BIND:
165 // if mount creates an output
166 if m.Output != int64(opspb.SkipOutput) {
167 // if it is readonly and not root then output is the input
168 if m.Readonly && ref != nil && m.Dest != opspb.RootMount {
169 p.OutputRefs = append(p.OutputRefs, MountRef{
170 MountIndex: i,
171 Ref: ref.Clone(),
172 })
173 } else {
174 // otherwise output and mount is the mutable child
175 active, err := makeMutable(m, ref)
176 if err != nil {
177 return p, err
178 }
179 mountable = active
180 p.OutputRefs = append(p.OutputRefs, MountRef{
181 MountIndex: i,
182 Ref: active,
183 })
184 }
185 } else if (!m.Readonly || ref == nil) && m.Dest != opspb.RootMount {
186 // this case is empty readonly scratch without output that is not really useful for anything but don't error
187 active, err := makeMutable(m, ref)
188 if err != nil {
189 return p, err
190 }
191 p.Actives = append(p.Actives, MountMutableRef{
192 MountIndex: i,
193 Ref: active,
194 })
195 mountable = active
196 }
197
198 case opspb.MountType_CACHE:
199 active, err := mm.MountableCache(ctx, m, ref, g)
200 if err != nil {
201 return p, err
202 }
203 mountable = active
204 p.Actives = append(p.Actives, MountMutableRef{
205 MountIndex: i,
206 Ref: active,
207 NoCommit: true,
208 })
209 if m.Output != int64(opspb.SkipOutput) && ref != nil {
210 p.OutputRefs = append(p.OutputRefs, MountRef{
211 MountIndex: i,
212 Ref: ref.Clone(),
213 })
214 }
215
216 case opspb.MountType_TMPFS:
217 mountable = mm.MountableTmpFS(m)
218 case opspb.MountType_SECRET:
219 var err error
220 mountable, err = mm.MountableSecret(ctx, m, g)
221 if err != nil {
222 return p, err
223 }
224 if mountable == nil {
225 continue
226 }
227 case opspb.MountType_SSH:
228 var err error
229 mountable, err = mm.MountableSSH(ctx, m, g)
230 if err != nil {
231 return p, err
232 }
233 if mountable == nil {
234 continue
235 }
236
237 default:
238 return p, errors.Errorf("mount type %s not implemented", m.MountType)
239 }
240
241 // validate that there is a mount
242 if mountable == nil {
243 return p, errors.Errorf("mount %s has no input", m.Dest)
244 }
245
246 // if dest is root we need mutable ref even if there is no output
247 if m.Dest == opspb.RootMount {
248 root := mountable
249 p.ReadonlyRootFS = m.Readonly
250 if m.Output == int64(opspb.SkipOutput) && p.ReadonlyRootFS {
251 active, err := makeMutable(m, ref)
252 if err != nil {
253 return p, err
254 }
255 p.Actives = append(p.Actives, MountMutableRef{
256 MountIndex: i,
257 Ref: active,
258 })
259 root = active
260 }
261 p.Root = MountWithSession(root, g)
262 } else {
263 mws := MountWithSession(mountable, g)
264 dest := m.Dest
265 if !system.IsAbs(filepath.Clean(dest), platform) {
266 dest = filepath.Join("/", cwd, dest)
267 }
268 mws.Dest = dest
269 mws.Readonly = m.Readonly
270 mws.Selector = m.Selector
271 p.Mounts = append(p.Mounts, mws)
272 }
273 }
274
275 // sort mounts so parents are mounted first
276 slices.SortFunc(p.Mounts, func(a, b executor.Mount) int {
277 return cmp.Compare(a.Dest, b.Dest)
278 })
279
280 return p, nil
281 }
282
283 type gatewayContainer struct {
284 id string
285 netMode opspb.NetMode
286 hostname string
287 extraHosts []executor.HostIP
288 platform *opspb.Platform
289 rootFS executor.Mount
290 mounts []executor.Mount
291 executor executor.Executor
292 sm *session.Manager
293 group session.Group
294 started bool
295 errGroup *errgroup.Group
296 mu sync.Mutex
297 cleanup []func() error
298 ctx context.Context
299 cancel func(error)
300 }
301
302 func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartRequest) (client.ContainerProcess, error) {
303 resize := make(chan executor.WinSize)
304 signal := make(chan syscall.Signal)
305 procInfo := executor.ProcessInfo{
306 Meta: executor.Meta{
307 Args: req.Args,
308 Env: req.Env,
309 User: req.User,
310 Cwd: req.Cwd,
311 Tty: req.Tty,
312 NetMode: gwCtr.netMode,
313 Hostname: gwCtr.hostname,
314 ExtraHosts: gwCtr.extraHosts,
315 SecurityMode: req.SecurityMode,
316 RemoveMountStubsRecursive: req.RemoveMountStubsRecursive,
317 },
318 Stdin: req.Stdin,
319 Stdout: req.Stdout,
320 Stderr: req.Stderr,
321 Resize: resize,
322 Signal: signal,
323 }
324 if procInfo.Meta.Cwd == "" {
325 procInfo.Meta.Cwd = "/"
326 }
327 procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "PATH", system.DefaultPathEnv(gwCtr.platform.OS))
328 if req.Tty {
329 procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "TERM", "xterm")
330 }
331
332 secretEnv, err := gwCtr.loadSecretEnv(ctx, req.SecretEnv)
333 if err != nil {
334 return nil, err
335 }
336 procInfo.Meta.Env = append(procInfo.Meta.Env, secretEnv...)
337
338 // mark that we have started on the first call to execProcess for this
339 // container, so that future calls will call Exec rather than Run
340 gwCtr.mu.Lock()
341 started := gwCtr.started
342 gwCtr.started = true
343 gwCtr.mu.Unlock()
344
345 eg, ctx := errgroup.WithContext(gwCtr.ctx)
346 gwProc := &gatewayContainerProcess{
347 resize: resize,
348 signal: signal,
349 errGroup: eg,
350 groupCtx: ctx,
351 }
352
353 if !started {
354 startedCh := make(chan struct{})
355 gwProc.errGroup.Go(func() error {
356 bklog.G(gwCtr.ctx).Debugf("Starting new container for %s with args: %q", gwCtr.id, procInfo.Meta.Args)
357 _, err := gwCtr.executor.Run(ctx, gwCtr.id, gwCtr.rootFS, gwCtr.mounts, procInfo, startedCh)
358 return stack.Enable(err)
359 })
360 select {
361 case <-ctx.Done():
362 case <-startedCh:
363 }
364 } else {
365 gwProc.errGroup.Go(func() error {
366 bklog.G(gwCtr.ctx).Debugf("Execing into container %s with args: %q", gwCtr.id, procInfo.Meta.Args)
367 err := gwCtr.executor.Exec(ctx, gwCtr.id, procInfo)
368 return stack.Enable(err)
369 })
370 }
371
372 gwCtr.errGroup.Go(gwProc.errGroup.Wait)
373
374 return gwProc, nil
375 }
376
377 func (gwCtr *gatewayContainer) loadSecretEnv(ctx context.Context, secretEnv []*opspb.SecretEnv) ([]string, error) {
378 out := make([]string, 0, len(secretEnv))
379 for _, sopt := range secretEnv {
380 id := sopt.ID
381 if id == "" {
382 return nil, errors.Errorf("secret ID missing for %q environment variable", sopt.Name)
383 }
384 var dt []byte
385 var err error
386 err = gwCtr.sm.Any(ctx, gwCtr.group, func(ctx context.Context, _ string, caller session.Caller) error {
387 dt, err = secrets.GetSecret(ctx, caller, id)
388 if err != nil {
389 return err
390 }
391 return nil
392 })
393 if err != nil && (!errors.Is(err, secrets.ErrNotFound) || !sopt.Optional) {
394 return nil, err
395 }
396 out = append(out, fmt.Sprintf("%s=%s", sopt.Name, string(dt)))
397 }
398 return out, nil
399 }
400
401 func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
402 gwCtr.mu.Lock()
403 defer gwCtr.mu.Unlock()
404 gwCtr.cancel(errors.WithStack(context.Canceled))
405 err1 := gwCtr.errGroup.Wait()
406
407 var err2 error
408 for i := len(gwCtr.cleanup) - 1; i >= 0; i-- { // call in LIFO order
409 err := gwCtr.cleanup[i]()
410 if err2 == nil {
411 err2 = err
412 }
413 }
414 gwCtr.cleanup = nil
415
416 if err1 != nil {
417 return stack.Enable(err1)
418 }
419 return stack.Enable(err2)
420 }
421
422 type gatewayContainerProcess struct {
423 errGroup *errgroup.Group
424 groupCtx context.Context
425 resize chan<- executor.WinSize
426 signal chan<- syscall.Signal
427 mu sync.Mutex
428 }
429
430 func (gwProc *gatewayContainerProcess) Wait() error {
431 err := stack.Enable(gwProc.errGroup.Wait())
432 gwProc.mu.Lock()
433 defer gwProc.mu.Unlock()
434 close(gwProc.resize)
435 close(gwProc.signal)
436 return err
437 }
438
439 func (gwProc *gatewayContainerProcess) Resize(ctx context.Context, size client.WinSize) error {
440 gwProc.mu.Lock()
441 defer gwProc.mu.Unlock()
442
443 // is the container done or should we proceed with sending event?
444 select {
445 case <-gwProc.groupCtx.Done():
446 return nil
447 case <-ctx.Done():
448 return nil
449 default:
450 }
451
452 // now we select on contexts again in case p.resize blocks b/c
453 // container no longer reading from it. In that case when
454 // the errgroup finishes we want to unblock on the write
455 // and exit
456 select {
457 case <-gwProc.groupCtx.Done():
458 case <-ctx.Done():
459 case gwProc.resize <- executor.WinSize{Cols: size.Cols, Rows: size.Rows}:
460 }
461 return nil
462 }
463
464 func (gwProc *gatewayContainerProcess) Signal(ctx context.Context, sig syscall.Signal) error {
465 gwProc.mu.Lock()
466 defer gwProc.mu.Unlock()
467
468 // is the container done or should we proceed with sending event?
469 select {
470 case <-gwProc.groupCtx.Done():
471 return nil
472 case <-ctx.Done():
473 return nil
474 default:
475 }
476
477 // now we select on contexts again in case p.signal blocks b/c
478 // container no longer reading from it. In that case when
479 // the errgroup finishes we want to unblock on the write
480 // and exit
481 select {
482 case <-gwProc.groupCtx.Done():
483 case <-ctx.Done():
484 case gwProc.signal <- sig:
485 }
486 return nil
487 }
488
489 func addDefaultEnvvar(env []string, k, v string) []string {
490 for _, e := range env {
491 if strings.HasPrefix(e, k+"=") {
492 return env
493 }
494 }
495 return append(env, k+"="+v)
496 }
497
498 func MountWithSession(m cache.Mountable, g session.Group) executor.Mount {
499 _, readonly := m.(cache.ImmutableRef)
500 return executor.Mount{
501 Src: &mountable{m: m, g: g},
502 Readonly: readonly,
503 }
504 }
505
506 type mountable struct {
507 m cache.Mountable
508 g session.Group
509 }
510
511 func (m *mountable) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
512 return m.m.Mount(ctx, readonly, m.g)
513 }