Debian Codegraph @

buildkit/frontend/gateway/grpcclient/client.go

raw
1 package grpcclient
2
3 import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net"
9 "os"
10 "strings"
11 "sync"
12 "syscall"
13 "time"
14
15 distreference "github.com/distribution/reference"
16 "github.com/moby/buildkit/client/llb"
17 "github.com/moby/buildkit/client/llb/sourceresolver"
18 "github.com/moby/buildkit/frontend/gateway/client"
19 pb "github.com/moby/buildkit/frontend/gateway/pb"
20 "github.com/moby/buildkit/identity"
21 opspb "github.com/moby/buildkit/solver/pb"
22 "github.com/moby/buildkit/util/apicaps"
23 "github.com/moby/buildkit/util/bklog"
24 "github.com/moby/buildkit/util/grpcerrors"
25 "github.com/moby/buildkit/util/imageutil"
26 "github.com/moby/sys/signal"
27 digest "github.com/opencontainers/go-digest"
28 "github.com/pkg/errors"
29 fstypes "github.com/tonistiigi/fsutil/types"
30 "golang.org/x/sync/errgroup"
31 spb "google.golang.org/genproto/googleapis/rpc/status"
32 "google.golang.org/grpc"
33 "google.golang.org/grpc/codes"
34 "google.golang.org/grpc/credentials/insecure"
35 "google.golang.org/grpc/status"
36 )
37
38 const frontendPrefix = "BUILDKIT_FRONTEND_OPT_"
39
40 type GrpcClient interface {
41 client.Client
42 Run(context.Context, client.BuildFunc) error
43 }
44
45 func New(ctx context.Context, opts map[string]string, session, product string, c pb.LLBBridgeClient, w []client.WorkerInfo) (GrpcClient, error) {
46 pingCtx, pingCancel := context.WithCancelCause(ctx)
47 pingCtx, _ = context.WithTimeoutCause(pingCtx, 15*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
48 defer pingCancel(errors.WithStack(context.Canceled))
49 resp, err := c.Ping(pingCtx, &pb.PingRequest{})
50 if err != nil {
51 return nil, err
52 }
53
54 if resp.FrontendAPICaps == nil {
55 resp.FrontendAPICaps = defaultCaps()
56 }
57
58 if resp.LLBCaps == nil {
59 resp.LLBCaps = defaultLLBCaps()
60 }
61
62 return &grpcClient{
63 client: c,
64 opts: opts,
65 sessionID: session,
66 workers: w,
67 product: product,
68 caps: pb.Caps.CapSet(resp.FrontendAPICaps),
69 llbCaps: opspb.Caps.CapSet(resp.LLBCaps),
70 requests: map[string]*pb.SolveRequest{},
71 execMsgs: newMessageForwarder(ctx, c),
72 }, nil
73 }
74
75 func current() (GrpcClient, error) {
76 if ep := product(); ep != "" {
77 apicaps.ExportedProduct = ep
78 }
79
80 ctx, conn, err := grpcClientConn(context.Background())
81 if err != nil {
82 return nil, err
83 }
84
85 return New(ctx, opts(), sessionID(), product(), pb.NewLLBBridgeClient(conn), workers())
86 }
87
88 func convertRef(ref client.Reference) (*pb.Ref, error) {
89 if ref == nil {
90 return &pb.Ref{}, nil
91 }
92 r, ok := ref.(*reference)
93 if !ok {
94 return nil, errors.Errorf("invalid return reference type %T", ref)
95 }
96 return &pb.Ref{Id: r.id, Def: r.def}, nil
97 }
98
99 func RunFromEnvironment(ctx context.Context, f client.BuildFunc) error {
100 client, err := current()
101 if err != nil {
102 return errors.Wrapf(err, "failed to initialize client from environment")
103 }
104 return client.Run(ctx, f)
105 }
106
107 func (c *grpcClient) Run(ctx context.Context, f client.BuildFunc) (retError error) {
108 export := c.caps.Supports(pb.CapReturnResult) == nil
109
110 var (
111 res *client.Result
112 err error
113 )
114 if export {
115 defer func() {
116 req := &pb.ReturnRequest{}
117 if retError == nil {
118 if res == nil {
119 res = client.NewResult()
120 }
121 pbRes := &pb.Result{
122 Metadata: res.Metadata,
123 }
124 if res.Refs != nil {
125 if c.caps.Supports(pb.CapProtoRefArray) == nil {
126 m := map[string]*pb.Ref{}
127 for k, r := range res.Refs {
128 pbRef, err := convertRef(r)
129 if err != nil {
130 retError = err
131 continue
132 }
133 m[k] = pbRef
134 }
135 pbRes.Result = &pb.Result_Refs{Refs: &pb.RefMap{Refs: m}}
136 } else {
137 // Server doesn't support the new wire format for refs, so we construct
138 // a deprecated result ref map.
139 m := map[string]string{}
140 for k, r := range res.Refs {
141 pbRef, err := convertRef(r)
142 if err != nil {
143 retError = err
144 continue
145 }
146 m[k] = pbRef.Id
147 }
148 pbRes.Result = &pb.Result_RefsDeprecated{RefsDeprecated: &pb.RefMapDeprecated{Refs: m}}
149 }
150 } else {
151 pbRef, err := convertRef(res.Ref)
152 if err != nil {
153 retError = err
154 } else {
155 if c.caps.Supports(pb.CapProtoRefArray) == nil {
156 pbRes.Result = &pb.Result_Ref{Ref: pbRef}
157 } else {
158 // Server doesn't support the new wire format for refs, so we construct
159 // a deprecated result ref.
160 pbRes.Result = &pb.Result_RefDeprecated{RefDeprecated: pbRef.Id}
161 }
162 }
163 }
164
165 if res.Attestations != nil {
166 attestations := map[string]*pb.Attestations{}
167 for k, as := range res.Attestations {
168 for _, a := range as {
169 pbAtt, err := client.AttestationToPB(&a)
170 if err != nil {
171 retError = err
172 continue
173 }
174 pbRef, err := convertRef(a.Ref)
175 if err != nil {
176 retError = err
177 continue
178 }
179 pbAtt.Ref = pbRef
180 if attestations[k] == nil {
181 attestations[k] = &pb.Attestations{}
182 }
183 attestations[k].Attestation = append(attestations[k].Attestation, pbAtt)
184 }
185 }
186 pbRes.Attestations = attestations
187 }
188
189 if retError == nil {
190 req.Result = pbRes
191 }
192 }
193 if retError != nil {
194 st, _ := status.FromError(grpcerrors.ToGRPC(ctx, retError))
195 stp := st.Proto()
196 req.Error = &spb.Status{
197 Code: stp.Code,
198 Message: stp.Message,
199 Details: stp.Details,
200 }
201 }
202 if _, err := c.client.Return(ctx, req); err != nil && retError == nil {
203 retError = err
204 }
205 }()
206 }
207
208 defer func() {
209 err = c.execMsgs.Release()
210 if err != nil && retError != nil {
211 retError = err
212 }
213 }()
214
215 if res, err = f(ctx, c); err != nil {
216 return err
217 }
218
219 if res == nil {
220 return nil
221 }
222
223 if err := c.caps.Supports(pb.CapReturnMap); len(res.Refs) > 1 && err != nil {
224 return err
225 }
226
227 if !export {
228 exportedAttrBytes, err := json.Marshal(res.Metadata)
229 if err != nil {
230 return errors.Wrapf(err, "failed to marshal return metadata")
231 }
232
233 req, err := c.requestForRef(res.Ref)
234 if err != nil {
235 return errors.Wrapf(err, "failed to find return ref")
236 }
237
238 req.Final = true
239 req.ExporterAttr = exportedAttrBytes
240
241 if _, err := c.client.Solve(ctx, req); err != nil {
242 return errors.Wrapf(err, "failed to solve")
243 }
244 }
245
246 return nil
247 }
248
249 // defaultCaps returns the capabilities that were implemented when capabilities
250 // support was added. This list is frozen and should never be changed.
251 func defaultCaps() []*apicaps.PBCap {
252 return []*apicaps.PBCap{
253 {ID: string(pb.CapSolveBase), Enabled: true},
254 {ID: string(pb.CapSolveInlineReturn), Enabled: true},
255 {ID: string(pb.CapResolveImage), Enabled: true},
256 {ID: string(pb.CapReadFile), Enabled: true},
257 }
258 }
259
260 // defaultLLBCaps returns the LLB capabilities that were implemented when capabilities
261 // support was added. This list is frozen and should never be changed.
262 func defaultLLBCaps() []*apicaps.PBCap {
263 return []*apicaps.PBCap{
264 {ID: string(opspb.CapSourceImage), Enabled: true},
265 {ID: string(opspb.CapSourceLocal), Enabled: true},
266 {ID: string(opspb.CapSourceLocalUnique), Enabled: true},
267 {ID: string(opspb.CapSourceLocalSessionID), Enabled: true},
268 {ID: string(opspb.CapSourceLocalIncludePatterns), Enabled: true},
269 {ID: string(opspb.CapSourceLocalFollowPaths), Enabled: true},
270 {ID: string(opspb.CapSourceLocalExcludePatterns), Enabled: true},
271 {ID: string(opspb.CapSourceLocalSharedKeyHint), Enabled: true},
272 {ID: string(opspb.CapSourceGit), Enabled: true},
273 {ID: string(opspb.CapSourceGitKeepDir), Enabled: true},
274 {ID: string(opspb.CapSourceGitFullURL), Enabled: true},
275 {ID: string(opspb.CapSourceHTTP), Enabled: true},
276 {ID: string(opspb.CapSourceHTTPChecksum), Enabled: true},
277 {ID: string(opspb.CapSourceHTTPPerm), Enabled: true},
278 {ID: string(opspb.CapSourceHTTPUIDGID), Enabled: true},
279 {ID: string(opspb.CapBuildOpLLBFileName), Enabled: true},
280 {ID: string(opspb.CapExecMetaBase), Enabled: true},
281 {ID: string(opspb.CapExecMetaProxy), Enabled: true},
282 {ID: string(opspb.CapExecMountBind), Enabled: true},
283 {ID: string(opspb.CapExecMountCache), Enabled: true},
284 {ID: string(opspb.CapExecMountCacheSharing), Enabled: true},
285 {ID: string(opspb.CapExecMountSelector), Enabled: true},
286 {ID: string(opspb.CapExecMountTmpfs), Enabled: true},
287 {ID: string(opspb.CapExecMountSecret), Enabled: true},
288 {ID: string(opspb.CapConstraints), Enabled: true},
289 {ID: string(opspb.CapPlatform), Enabled: true},
290 {ID: string(opspb.CapMetaIgnoreCache), Enabled: true},
291 {ID: string(opspb.CapMetaDescription), Enabled: true},
292 {ID: string(opspb.CapMetaExportCache), Enabled: true},
293 }
294 }
295
296 type grpcClient struct {
297 client pb.LLBBridgeClient
298 opts map[string]string
299 sessionID string
300 product string
301 workers []client.WorkerInfo
302 caps apicaps.CapSet
303 llbCaps apicaps.CapSet
304 requests map[string]*pb.SolveRequest
305 execMsgs *messageForwarder
306 }
307
308 func (c *grpcClient) requestForRef(ref client.Reference) (*pb.SolveRequest, error) {
309 emptyReq := &pb.SolveRequest{
310 Definition: &opspb.Definition{},
311 }
312 if ref == nil {
313 return emptyReq, nil
314 }
315 r, ok := ref.(*reference)
316 if !ok {
317 return nil, errors.Errorf("return reference has invalid type %T", ref)
318 }
319 if r.id == "" {
320 return emptyReq, nil
321 }
322 req, ok := c.requests[r.id]
323 if !ok {
324 return nil, errors.Errorf("did not find request for return reference %s", r.id)
325 }
326 return req, nil
327 }
328
329 func (c *grpcClient) Warn(ctx context.Context, dgst digest.Digest, msg string, opts client.WarnOpts) error {
330 _, err := c.client.Warn(ctx, &pb.WarnRequest{
331 Digest: string(dgst),
332 Level: int64(opts.Level),
333 Short: []byte(msg),
334 Info: opts.SourceInfo,
335 Ranges: opts.Range,
336 Detail: opts.Detail,
337 Url: opts.URL,
338 })
339 return err
340 }
341
342 func (c *grpcClient) Solve(ctx context.Context, creq client.SolveRequest) (res *client.Result, err error) {
343 if creq.Definition != nil {
344 for _, md := range creq.Definition.Metadata {
345 for cap := range md.Caps {
346 if err := c.llbCaps.Supports(apicaps.CapID(cap)); err != nil {
347 return nil, err
348 }
349 }
350 }
351 }
352 var cacheImports []*pb.CacheOptionsEntry
353 for _, im := range creq.CacheImports {
354 cacheImports = append(cacheImports, &pb.CacheOptionsEntry{
355 Type: im.Type,
356 Attrs: im.Attrs,
357 })
358 }
359
360 // these options are added by go client in solve()
361 if _, ok := creq.FrontendOpt["cache-imports"]; !ok {
362 if v, ok := c.opts["cache-imports"]; ok {
363 if creq.FrontendOpt == nil {
364 creq.FrontendOpt = map[string]string{}
365 }
366 creq.FrontendOpt["cache-imports"] = v
367 }
368 }
369 if _, ok := creq.FrontendOpt["cache-from"]; !ok {
370 if v, ok := c.opts["cache-from"]; ok {
371 if creq.FrontendOpt == nil {
372 creq.FrontendOpt = map[string]string{}
373 }
374 creq.FrontendOpt["cache-from"] = v
375 }
376 }
377
378 req := &pb.SolveRequest{
379 Definition: creq.Definition,
380 Frontend: creq.Frontend,
381 FrontendOpt: creq.FrontendOpt,
382 FrontendInputs: creq.FrontendInputs,
383 AllowResultReturn: true,
384 AllowResultArrayRef: true,
385 CacheImports: cacheImports,
386 SourcePolicies: creq.SourcePolicies,
387 }
388
389 // backwards compatibility with inline return
390 if c.caps.Supports(pb.CapReturnResult) != nil {
391 req.ExporterAttr = []byte("{}")
392 }
393
394 if creq.Evaluate {
395 if c.caps.Supports(pb.CapGatewayEvaluateSolve) == nil {
396 req.Evaluate = creq.Evaluate
397 } else {
398 // If evaluate is not supported, fallback to running Stat(".") in
399 // order to trigger an evaluation of the result.
400 defer func() {
401 if res == nil {
402 return
403 }
404 err = res.EachRef(func(ref client.Reference) error {
405 _, err := ref.StatFile(ctx, client.StatRequest{Path: "."})
406 return err
407 })
408 }()
409 }
410 }
411
412 resp, err := c.client.Solve(ctx, req)
413 if err != nil {
414 return nil, err
415 }
416
417 res = client.NewResult()
418 if resp.Result == nil {
419 if id := resp.Ref; id != "" {
420 c.requests[id] = req
421 }
422 res.SetRef(&reference{id: resp.Ref, c: c})
423 } else {
424 res.Metadata = resp.Result.Metadata
425 switch pbRes := resp.Result.Result.(type) {
426 case *pb.Result_RefDeprecated:
427 if id := pbRes.RefDeprecated; id != "" {
428 res.SetRef(&reference{id: id, c: c})
429 }
430 case *pb.Result_RefsDeprecated:
431 for k, v := range pbRes.RefsDeprecated.Refs {
432 var ref client.Reference
433 if v != "" {
434 ref = &reference{id: v, c: c}
435 }
436 res.AddRef(k, ref)
437 }
438 case *pb.Result_Ref:
439 if pbRes.Ref.Id != "" {
440 res.SetRef(newReference(c, pbRes.Ref))
441 }
442 case *pb.Result_Refs:
443 for k, v := range pbRes.Refs.Refs {
444 var ref client.Reference
445 if v.Id != "" {
446 ref = newReference(c, v)
447 }
448 res.AddRef(k, ref)
449 }
450 }
451
452 if resp.Result.Attestations != nil {
453 for p, as := range resp.Result.Attestations {
454 for _, a := range as.Attestation {
455 att, err := client.AttestationFromPB[client.Reference](a)
456 if err != nil {
457 return nil, err
458 }
459 if a.Ref.Id != "" {
460 att.Ref = newReference(c, a.Ref)
461 }
462 res.AddAttestation(p, *att)
463 }
464 }
465 }
466 }
467
468 return res, nil
469 }
470
471 func (c *grpcClient) ResolveSourceMetadata(ctx context.Context, op *opspb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
472 if c.caps.Supports(pb.CapSourceMetaResolver) != nil {
473 var ref string
474 if v, ok := strings.CutPrefix(op.Identifier, "docker-image://"); ok {
475 ref = v
476 } else if v, ok := strings.CutPrefix(op.Identifier, "oci-layout://"); ok {
477 ref = v
478 } else {
479 return &sourceresolver.MetaResponse{Op: op}, nil
480 }
481 retRef, dgst, config, err := c.ResolveImageConfig(ctx, ref, opt)
482 if err != nil {
483 return nil, err
484 }
485 if strings.HasPrefix(op.Identifier, "docker-image://") {
486 op.Identifier = "docker-image://" + retRef
487 } else if strings.HasPrefix(op.Identifier, "oci-layout://") {
488 op.Identifier = "oci-layout://" + retRef
489 }
490
491 return &sourceresolver.MetaResponse{
492 Op: op,
493 Image: &sourceresolver.ResolveImageResponse{
494 Digest: dgst,
495 Config: config,
496 },
497 }, nil
498 }
499
500 var p *opspb.Platform
501 if platform := opt.Platform; platform != nil {
502 p = &opspb.Platform{
503 OS: platform.OS,
504 Architecture: platform.Architecture,
505 Variant: platform.Variant,
506 OSVersion: platform.OSVersion,
507 OSFeatures: platform.OSFeatures,
508 }
509 }
510
511 req := &pb.ResolveSourceMetaRequest{
512 Source: op,
513 Platform: p,
514 LogName: opt.LogName,
515 SourcePolicies: opt.SourcePolicies,
516 }
517 resp, err := c.client.ResolveSourceMeta(ctx, req)
518 if err != nil {
519 return nil, err
520 }
521
522 r := &sourceresolver.MetaResponse{
523 Op: resp.Source,
524 }
525 if resp.Image != nil {
526 r.Image = &sourceresolver.ResolveImageResponse{
527 Digest: digest.Digest(resp.Image.Digest),
528 Config: resp.Image.Config,
529 }
530 }
531 return r, nil
532 }
533
534 func (c *grpcClient) resolveImageConfigViaSourceMetadata(ctx context.Context, ref string, opt sourceresolver.Opt, p *opspb.Platform) (string, digest.Digest, []byte, error) {
535 op := &opspb.SourceOp{
536 Identifier: "docker-image://" + ref,
537 }
538 if opt.OCILayoutOpt != nil {
539 named, err := distreference.ParseNormalizedNamed(ref)
540 if err != nil {
541 return "", "", nil, err
542 }
543 op.Identifier = "oci-layout://" + named.String()
544 op.Attrs = map[string]string{
545 opspb.AttrOCILayoutSessionID: opt.OCILayoutOpt.Store.SessionID,
546 opspb.AttrOCILayoutStoreID: opt.OCILayoutOpt.Store.StoreID,
547 }
548 }
549
550 req := &pb.ResolveSourceMetaRequest{
551 Source: op,
552 Platform: p,
553 LogName: opt.LogName,
554 SourcePolicies: opt.SourcePolicies,
555 }
556 resp, err := c.client.ResolveSourceMeta(ctx, req)
557 if err != nil {
558 return "", "", nil, err
559 }
560 if resp.Image == nil {
561 return "", "", nil, &imageutil.ResolveToNonImageError{Ref: ref, Updated: resp.Source.Identifier}
562 }
563 ref = strings.TrimPrefix(resp.Source.Identifier, "docker-image://")
564 ref = strings.TrimPrefix(ref, "oci-layout://")
565 return ref, digest.Digest(resp.Image.Digest), resp.Image.Config, nil
566 }
567
568 func (c *grpcClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
569 var p *opspb.Platform
570 if platform := opt.Platform; platform != nil {
571 p = &opspb.Platform{
572 OS: platform.OS,
573 Architecture: platform.Architecture,
574 Variant: platform.Variant,
575 OSVersion: platform.OSVersion,
576 OSFeatures: platform.OSFeatures,
577 }
578 }
579
580 if c.caps.Supports(pb.CapSourceMetaResolver) == nil {
581 return c.resolveImageConfigViaSourceMetadata(ctx, ref, opt, p)
582 }
583
584 req := &pb.ResolveImageConfigRequest{
585 Ref: ref,
586 LogName: opt.LogName,
587 SourcePolicies: opt.SourcePolicies,
588 Platform: p,
589 }
590 if iopt := opt.ImageOpt; iopt != nil {
591 req.ResolveMode = iopt.ResolveMode
592 req.ResolverType = int32(sourceresolver.ResolverTypeRegistry)
593 }
594
595 if iopt := opt.OCILayoutOpt; iopt != nil {
596 req.ResolverType = int32(sourceresolver.ResolverTypeOCILayout)
597 req.StoreID = iopt.Store.StoreID
598 req.SessionID = iopt.Store.SessionID
599 }
600
601 resp, err := c.client.ResolveImageConfig(ctx, req)
602 if err != nil {
603 return "", "", nil, err
604 }
605 newRef := resp.Ref
606 if newRef == "" {
607 // No ref returned, use the original one.
608 // This could occur if the version of buildkitd is too old.
609 newRef = ref
610 }
611 return newRef, digest.Digest(resp.Digest), resp.Config, nil
612 }
613
614 func (c *grpcClient) BuildOpts() client.BuildOpts {
615 return client.BuildOpts{
616 Opts: c.opts,
617 SessionID: c.sessionID,
618 Workers: c.workers,
619 Product: c.product,
620 LLBCaps: c.llbCaps,
621 Caps: c.caps,
622 }
623 }
624
625 func (c *grpcClient) CurrentFrontend() (*llb.State, error) {
626 fp := "/run/config/buildkit/metadata/frontend.bin"
627 if _, err := os.Stat(fp); err != nil {
628 return nil, nil
629 }
630 dt, err := os.ReadFile(fp)
631 if err != nil {
632 return nil, err
633 }
634 var def opspb.Definition
635 if err := def.UnmarshalVT(dt); err != nil {
636 return nil, err
637 }
638 op, err := llb.NewDefinitionOp(&def)
639 if err != nil {
640 return nil, err
641 }
642 st := llb.NewState(op)
643 return &st, nil
644 }
645
646 func (c *grpcClient) Inputs(ctx context.Context) (map[string]llb.State, error) {
647 err := c.caps.Supports(pb.CapFrontendInputs)
648 if err != nil {
649 return nil, err
650 }
651
652 resp, err := c.client.Inputs(ctx, &pb.InputsRequest{})
653 if err != nil {
654 return nil, err
655 }
656
657 inputs := make(map[string]llb.State)
658 for key, def := range resp.Definitions {
659 op, err := llb.NewDefinitionOp(def)
660 if err != nil {
661 return nil, err
662 }
663 inputs[key] = llb.NewState(op)
664 }
665 return inputs, nil
666 }
667
668 // procMessageForwarder is created per container process to act as the
669 // communication channel between the process and the ExecProcess message
670 // stream.
671 type procMessageForwarder struct {
672 done chan struct{}
673 closeOnce sync.Once
674 msgs chan *pb.ExecMessage
675 }
676
677 func newProcMessageForwarder() *procMessageForwarder {
678 return &procMessageForwarder{
679 done: make(chan struct{}),
680 msgs: make(chan *pb.ExecMessage),
681 }
682 }
683
684 func (b *procMessageForwarder) Send(ctx context.Context, m *pb.ExecMessage) {
685 select {
686 case <-ctx.Done():
687 case <-b.done:
688 b.closeOnce.Do(func() {
689 close(b.msgs)
690 })
691 case b.msgs <- m:
692 }
693 }
694
695 func (b *procMessageForwarder) Recv(ctx context.Context) (m *pb.ExecMessage, ok bool) {
696 select {
697 case <-ctx.Done():
698 return nil, true
699 case <-b.done:
700 return nil, false
701 case m = <-b.msgs:
702 return m, true
703 }
704 }
705
706 func (b *procMessageForwarder) Close() {
707 close(b.done)
708 b.Recv(context.Background()) // flush any messages in queue
709 b.Send(context.Background(), nil) // ensure channel is closed
710 }
711
712 // messageForwarder manages a single grpc stream for ExecProcess to facilitate
713 // a pub/sub message channel for each new process started from the client
714 // connection.
715 type messageForwarder struct {
716 client pb.LLBBridgeClient
717 ctx context.Context
718 cancel func(error)
719 eg *errgroup.Group
720 mu sync.Mutex
721 pids map[string]*procMessageForwarder
722 stream pb.LLBBridge_ExecProcessClient
723 // startOnce used to only start the exec message forwarder once,
724 // so we only have one exec stream per client
725 startOnce sync.Once
726 // startErr tracks the error when initializing the stream, it will
727 // be returned on subsequent calls to Start
728 startErr error
729 }
730
731 func newMessageForwarder(ctx context.Context, client pb.LLBBridgeClient) *messageForwarder {
732 ctx, cancel := context.WithCancelCause(ctx)
733 eg, ctx := errgroup.WithContext(ctx)
734 return &messageForwarder{
735 client: client,
736 pids: map[string]*procMessageForwarder{},
737 ctx: ctx,
738 cancel: cancel,
739 eg: eg,
740 }
741 }
742
743 func (m *messageForwarder) Start() (err error) {
744 defer func() {
745 if err != nil {
746 m.startErr = err
747 }
748 }()
749
750 if m.startErr != nil {
751 return m.startErr
752 }
753
754 m.startOnce.Do(func() {
755 m.stream, err = m.client.ExecProcess(m.ctx)
756 if err != nil {
757 return
758 }
759 m.eg.Go(func() error {
760 for {
761 msg, err := m.stream.Recv()
762 if errors.Is(err, io.EOF) || grpcerrors.Code(err) == codes.Canceled {
763 return nil
764 }
765 bklog.G(m.ctx).Debugf("|<--- %s", debugMessage(msg))
766
767 if err != nil {
768 return err
769 }
770
771 m.mu.Lock()
772 msgs, ok := m.pids[msg.ProcessID]
773 m.mu.Unlock()
774
775 if !ok {
776 bklog.G(m.ctx).Debugf("Received exec message for unregistered process: %s", msg.String())
777 continue
778 }
779 msgs.Send(m.ctx, msg)
780 }
781 })
782 })
783 return err
784 }
785
786 func debugMessage(msg *pb.ExecMessage) string {
787 switch m := msg.GetInput().(type) {
788 case *pb.ExecMessage_Init:
789 return fmt.Sprintf("Init Message %s", msg.ProcessID)
790 case *pb.ExecMessage_File:
791 if m.File.EOF {
792 return fmt.Sprintf("File Message %s, fd=%d, EOF", msg.ProcessID, m.File.Fd)
793 }
794 return fmt.Sprintf("File Message %s, fd=%d, %d bytes", msg.ProcessID, m.File.Fd, len(m.File.Data))
795 case *pb.ExecMessage_Resize:
796 return fmt.Sprintf("Resize Message %s", msg.ProcessID)
797 case *pb.ExecMessage_Signal:
798 return fmt.Sprintf("Signal Message %s: %s", msg.ProcessID, m.Signal.Name)
799 case *pb.ExecMessage_Started:
800 return fmt.Sprintf("Started Message %s", msg.ProcessID)
801 case *pb.ExecMessage_Exit:
802 return fmt.Sprintf("Exit Message %s, code=%d, err=%s", msg.ProcessID, m.Exit.Code, m.Exit.Error)
803 case *pb.ExecMessage_Done:
804 return fmt.Sprintf("Done Message %s", msg.ProcessID)
805 }
806 return fmt.Sprintf("Unknown Message %s", msg.String())
807 }
808
809 func (m *messageForwarder) Send(msg *pb.ExecMessage) error {
810 m.mu.Lock()
811 _, ok := m.pids[msg.ProcessID]
812 defer m.mu.Unlock()
813 if !ok {
814 return errors.Errorf("process %s has ended, not sending message %#v", msg.ProcessID, msg.Input)
815 }
816 bklog.G(m.ctx).Debugf("|---> %s", debugMessage(msg))
817 return m.stream.Send(msg)
818 }
819
820 func (m *messageForwarder) Release() error {
821 m.cancel(errors.WithStack(context.Canceled))
822 return m.eg.Wait()
823 }
824
825 func (m *messageForwarder) Register(pid string) *procMessageForwarder {
826 m.mu.Lock()
827 defer m.mu.Unlock()
828 sender := newProcMessageForwarder()
829 m.pids[pid] = sender
830 return sender
831 }
832
833 func (m *messageForwarder) Deregister(pid string) {
834 m.mu.Lock()
835 defer m.mu.Unlock()
836 sender, ok := m.pids[pid]
837 if !ok {
838 return
839 }
840 delete(m.pids, pid)
841 sender.Close()
842 }
843
844 type msgWriter struct {
845 mux *messageForwarder
846 fd uint32
847 processID string
848 }
849
850 func (w *msgWriter) Write(msg []byte) (int, error) {
851 err := w.mux.Send(&pb.ExecMessage{
852 ProcessID: w.processID,
853 Input: &pb.ExecMessage_File{
854 File: &pb.FdMessage{
855 Fd: w.fd,
856 Data: msg,
857 },
858 },
859 })
860 if err != nil {
861 return 0, err
862 }
863 return len(msg), nil
864 }
865
866 func (c *grpcClient) NewContainer(ctx context.Context, req client.NewContainerRequest) (client.Container, error) {
867 err := c.caps.Supports(pb.CapGatewayExec)
868 if err != nil {
869 return nil, err
870 }
871 id := identity.NewID()
872 var mounts []*opspb.Mount
873 for _, m := range req.Mounts {
874 resultID := m.ResultID
875 if m.Ref != nil {
876 ref, ok := m.Ref.(*reference)
877 if !ok {
878 return nil, errors.Errorf("unexpected type for reference, got %T", m.Ref)
879 }
880 resultID = ref.id
881 }
882 mounts = append(mounts, &opspb.Mount{
883 Dest: m.Dest,
884 Selector: m.Selector,
885 Readonly: m.Readonly,
886 MountType: m.MountType,
887 ResultID: resultID,
888 CacheOpt: m.CacheOpt,
889 SecretOpt: m.SecretOpt,
890 SSHOpt: m.SSHOpt,
891 })
892 }
893
894 bklog.G(ctx).Debugf("|---> NewContainer %s", id)
895 _, err = c.client.NewContainer(ctx, &pb.NewContainerRequest{
896 ContainerID: id,
897 Mounts: mounts,
898 Platform: req.Platform,
899 Constraints: req.Constraints,
900 Network: req.NetMode,
901 ExtraHosts: req.ExtraHosts,
902 Hostname: req.Hostname,
903 })
904 if err != nil {
905 return nil, err
906 }
907
908 // ensure message forwarder is started, only sets up stream first time called
909 err = c.execMsgs.Start()
910 if err != nil {
911 return nil, err
912 }
913
914 return &container{
915 client: c.client,
916 caps: c.caps,
917 id: id,
918 execMsgs: c.execMsgs,
919 }, nil
920 }
921
922 type container struct {
923 client pb.LLBBridgeClient
924 caps apicaps.CapSet
925 id string
926 execMsgs *messageForwarder
927 }
928
929 func (ctr *container) Start(ctx context.Context, req client.StartRequest) (client.ContainerProcess, error) {
930 pid := fmt.Sprintf("%s:%s", ctr.id, identity.NewID())
931 msgs := ctr.execMsgs.Register(pid)
932
933 if len(req.SecretEnv) > 0 {
934 if err := ctr.caps.Supports(pb.CapGatewayExecSecretEnv); err != nil {
935 return nil, err
936 }
937 }
938
939 init := &pb.InitMessage{
940 ContainerID: ctr.id,
941 Meta: &opspb.Meta{
942 Args: req.Args,
943 Env: req.Env,
944 Cwd: req.Cwd,
945 User: req.User,
946 },
947 Tty: req.Tty,
948 Security: req.SecurityMode,
949 Secretenv: req.SecretEnv,
950 }
951 init.Meta.RemoveMountStubsRecursive = req.RemoveMountStubsRecursive
952 if req.Stdin != nil {
953 init.Fds = append(init.Fds, 0)
954 }
955 if req.Stdout != nil {
956 init.Fds = append(init.Fds, 1)
957 }
958 if req.Stderr != nil {
959 init.Fds = append(init.Fds, 2)
960 }
961
962 err := ctr.execMsgs.Send(&pb.ExecMessage{
963 ProcessID: pid,
964 Input: &pb.ExecMessage_Init{
965 Init: init,
966 },
967 })
968 if err != nil {
969 return nil, err
970 }
971
972 msg, _ := msgs.Recv(ctx)
973 if msg == nil {
974 return nil, errors.Errorf("failed to receive started message")
975 }
976 started := msg.GetStarted()
977 if started == nil {
978 return nil, errors.Errorf("expecting started message, got %T", msg.GetInput())
979 }
980
981 eg, ctx := errgroup.WithContext(ctx)
982 done := make(chan struct{})
983
984 ctrProc := &containerProcess{
985 execMsgs: ctr.execMsgs,
986 id: pid,
987 eg: eg,
988 }
989
990 var stdinReader *io.PipeReader
991 ctrProc.eg.Go(func() error {
992 <-done
993 if stdinReader != nil {
994 return stdinReader.Close()
995 }
996 return nil
997 })
998
999 if req.Stdin != nil {
1000 var stdinWriter io.WriteCloser
1001 stdinReader, stdinWriter = io.Pipe()
1002 // This go routine is intentionally not part of the errgroup because
1003 // if os.Stdin is used for req.Stdin then this will block until
1004 // the user closes the input, which will likely be after we are done
1005 // with the container, so we can't Wait on it.
1006 go func() {
1007 io.Copy(stdinWriter, req.Stdin)
1008 stdinWriter.Close()
1009 }()
1010
1011 ctrProc.eg.Go(func() error {
1012 m := &msgWriter{
1013 mux: ctr.execMsgs,
1014 processID: pid,
1015 fd: 0,
1016 }
1017 _, err := io.Copy(m, stdinReader)
1018 // ignore ErrClosedPipe, it is EOF for our usage.
1019 if err != nil && !errors.Is(err, io.ErrClosedPipe) {
1020 return err
1021 }
1022 // not an error so must be eof
1023 return ctr.execMsgs.Send(&pb.ExecMessage{
1024 ProcessID: pid,
1025 Input: &pb.ExecMessage_File{
1026 File: &pb.FdMessage{
1027 Fd: 0,
1028 EOF: true,
1029 },
1030 },
1031 })
1032 })
1033 }
1034
1035 ctrProc.eg.Go(func() error {
1036 var closeDoneOnce sync.Once
1037 var exitError error
1038 for {
1039 msg, ok := msgs.Recv(ctx)
1040 if !ok {
1041 // no more messages, return
1042 return exitError
1043 }
1044
1045 if msg == nil {
1046 // empty message from ctx cancel, so just start shutting down
1047 // input
1048 closeDoneOnce.Do(func() {
1049 close(done)
1050 })
1051 return context.Cause(ctx)
1052 }
1053
1054 if file := msg.GetFile(); file != nil {
1055 var out io.WriteCloser
1056 switch file.Fd {
1057 case 1:
1058 out = req.Stdout
1059 case 2:
1060 out = req.Stderr
1061 }
1062 if out == nil {
1063 // if things are plumbed correctly this should never happen
1064 return errors.Errorf("missing writer for output fd %d", file.Fd)
1065 }
1066 if len(file.Data) > 0 {
1067 _, err := out.Write(file.Data)
1068 if err != nil {
1069 return err
1070 }
1071 }
1072 } else if exit := msg.GetExit(); exit != nil {
1073 // capture exit message to exitError so we can return it after
1074 // the server sends the Done message
1075 closeDoneOnce.Do(func() {
1076 close(done)
1077 })
1078 if exit.Code == 0 {
1079 continue
1080 }
1081 exitError = grpcerrors.FromGRPC(status.ErrorProto(&spb.Status{
1082 Code: exit.Error.Code,
1083 Message: exit.Error.Message,
1084 Details: exit.Error.Details,
1085 }))
1086 if exit.Code != pb.UnknownExitStatus {
1087 exitError = &pb.ExitError{ExitCode: exit.Code, Err: exitError}
1088 }
1089 } else if serverDone := msg.GetDone(); serverDone != nil {
1090 return exitError
1091 } else {
1092 return errors.Errorf("unexpected Exec Message for pid %s: %T", pid, msg.GetInput())
1093 }
1094 }
1095 })
1096
1097 return ctrProc, nil
1098 }
1099
1100 func (ctr *container) Release(ctx context.Context) error {
1101 bklog.G(ctx).Debugf("|---> ReleaseContainer %s", ctr.id)
1102 _, err := ctr.client.ReleaseContainer(ctx, &pb.ReleaseContainerRequest{
1103 ContainerID: ctr.id,
1104 })
1105 return err
1106 }
1107
1108 type containerProcess struct {
1109 execMsgs *messageForwarder
1110 id string
1111 eg *errgroup.Group
1112 }
1113
1114 func (ctrProc *containerProcess) Wait() error {
1115 defer ctrProc.execMsgs.Deregister(ctrProc.id)
1116 return ctrProc.eg.Wait()
1117 }
1118
1119 func (ctrProc *containerProcess) Resize(_ context.Context, size client.WinSize) error {
1120 return ctrProc.execMsgs.Send(&pb.ExecMessage{
1121 ProcessID: ctrProc.id,
1122 Input: &pb.ExecMessage_Resize{
1123 Resize: &pb.ResizeMessage{
1124 Cols: size.Cols,
1125 Rows: size.Rows,
1126 },
1127 },
1128 })
1129 }
1130
1131 var sigToName = map[syscall.Signal]string{}
1132
1133 func init() {
1134 for name, value := range signal.SignalMap {
1135 sigToName[value] = name
1136 }
1137 }
1138
1139 func (ctrProc *containerProcess) Signal(_ context.Context, sig syscall.Signal) error {
1140 name := sigToName[sig]
1141 if name == "" {
1142 return errors.Errorf("unknown signal %v", sig)
1143 }
1144 return ctrProc.execMsgs.Send(&pb.ExecMessage{
1145 ProcessID: ctrProc.id,
1146 Input: &pb.ExecMessage_Signal{
1147 Signal: &pb.SignalMessage{
1148 Name: name,
1149 },
1150 },
1151 })
1152 }
1153
1154 type reference struct {
1155 c *grpcClient
1156 id string
1157 def *opspb.Definition
1158 }
1159
1160 func newReference(c *grpcClient, ref *pb.Ref) *reference {
1161 return &reference{c: c, id: ref.Id, def: ref.Def}
1162 }
1163
1164 func (r *reference) ToState() (st llb.State, err error) {
1165 err = r.c.caps.Supports(pb.CapReferenceOutput)
1166 if err != nil {
1167 return st, err
1168 }
1169
1170 if r.def == nil {
1171 return st, errors.Errorf("gateway did not return reference with definition")
1172 }
1173
1174 defop, err := llb.NewDefinitionOp(r.def)
1175 if err != nil {
1176 return st, err
1177 }
1178
1179 return llb.NewState(defop), nil
1180 }
1181
1182 func (r *reference) Evaluate(ctx context.Context) error {
1183 req := &pb.EvaluateRequest{Ref: r.id}
1184 _, err := r.c.client.Evaluate(ctx, req)
1185 if err != nil {
1186 return err
1187 }
1188 return nil
1189 }
1190
1191 func (r *reference) ReadFile(ctx context.Context, req client.ReadRequest) ([]byte, error) {
1192 rfr := &pb.ReadFileRequest{FilePath: req.Filename, Ref: r.id}
1193 if r := req.Range; r != nil {
1194 rfr.Range = &pb.FileRange{
1195 Offset: int64(r.Offset),
1196 Length: int64(r.Length),
1197 }
1198 }
1199 resp, err := r.c.client.ReadFile(ctx, rfr)
1200 if err != nil {
1201 return nil, err
1202 }
1203 return resp.Data, nil
1204 }
1205
1206 func (r *reference) ReadDir(ctx context.Context, req client.ReadDirRequest) ([]*fstypes.Stat, error) {
1207 if err := r.c.caps.Supports(pb.CapReadDir); err != nil {
1208 return nil, err
1209 }
1210 rdr := &pb.ReadDirRequest{
1211 DirPath: req.Path,
1212 IncludePattern: req.IncludePattern,
1213 Ref: r.id,
1214 }
1215 resp, err := r.c.client.ReadDir(ctx, rdr)
1216 if err != nil {
1217 return nil, err
1218 }
1219 return resp.Entries, nil
1220 }
1221
1222 func (r *reference) StatFile(ctx context.Context, req client.StatRequest) (*fstypes.Stat, error) {
1223 if err := r.c.caps.Supports(pb.CapStatFile); err != nil {
1224 return nil, err
1225 }
1226 rdr := &pb.StatFileRequest{
1227 Path: req.Path,
1228 Ref: r.id,
1229 }
1230 resp, err := r.c.client.StatFile(ctx, rdr)
1231 if err != nil {
1232 return nil, err
1233 }
1234 return resp.Stat, nil
1235 }
1236
1237 func grpcClientConn(ctx context.Context) (context.Context, *grpc.ClientConn, error) {
1238 dialOpts := []grpc.DialOption{
1239 grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
1240 return stdioConn(), nil
1241 }),
1242 grpc.WithTransportCredentials(insecure.NewCredentials()),
1243 grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor),
1244 grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor),
1245 grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(16 << 20)),
1246 grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(16 << 20)),
1247 }
1248
1249 //nolint:staticcheck // ignore SA1019 NewClient has different behavior and needs to be tested
1250 cc, err := grpc.DialContext(ctx, "localhost", dialOpts...)
1251 if err != nil {
1252 return ctx, nil, errors.Wrap(err, "failed to create grpc client")
1253 }
1254
1255 ctx, cancel := context.WithCancelCause(ctx)
1256 _ = cancel
1257 // go monitorHealth(ctx, cc, cancel)
1258
1259 return ctx, cc, nil
1260 }
1261
1262 func stdioConn() net.Conn {
1263 return &conn{os.Stdin, os.Stdout, os.Stdout}
1264 }
1265
1266 type conn struct {
1267 io.Reader
1268 io.Writer
1269 io.Closer
1270 }
1271
1272 func (s *conn) LocalAddr() net.Addr {
1273 return dummyAddr{}
1274 }
1275
1276 func (s *conn) RemoteAddr() net.Addr {
1277 return dummyAddr{}
1278 }
1279
1280 func (s *conn) SetDeadline(t time.Time) error {
1281 return nil
1282 }
1283
1284 func (s *conn) SetReadDeadline(t time.Time) error {
1285 return nil
1286 }
1287
1288 func (s *conn) SetWriteDeadline(t time.Time) error {
1289 return nil
1290 }
1291
1292 type dummyAddr struct{}
1293
1294 func (d dummyAddr) Network() string {
1295 return "pipe"
1296 }
1297
1298 func (d dummyAddr) String() string {
1299 return "localhost"
1300 }
1301
1302 func opts() map[string]string {
1303 opts := map[string]string{}
1304 for _, env := range os.Environ() {
1305 parts := strings.SplitN(env, "=", 2)
1306 k := parts[0]
1307 v := ""
1308 if len(parts) == 2 {
1309 v = parts[1]
1310 }
1311 if !strings.HasPrefix(k, frontendPrefix) {
1312 continue
1313 }
1314 parts = strings.SplitN(v, "=", 2)
1315 v = ""
1316 if len(parts) == 2 {
1317 v = parts[1]
1318 }
1319 opts[parts[0]] = v
1320 }
1321 return opts
1322 }
1323
1324 func sessionID() string {
1325 return os.Getenv("BUILDKIT_SESSION_ID")
1326 }
1327
1328 func workers() []client.WorkerInfo {
1329 var c []client.WorkerInfo
1330 if err := json.Unmarshal([]byte(os.Getenv("BUILDKIT_WORKERS")), &c); err != nil {
1331 return nil
1332 }
1333 return c
1334 }
1335
1336 func product() string {
1337 return os.Getenv("BUILDKIT_EXPORTEDPRODUCT")
1338 }