Debian Codegraph @

launcher/spec/launch_spec.go

raw
1 // Package spec contains definition of some basic container launch specs needed to
2 // launch a container, provided by the operator.
3 package spec
4
5 import (
6 "context"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "os"
11 "os/exec"
12 "path"
13 "strconv"
14 "strings"
15
16 "cloud.google.com/go/compute/metadata"
17
18 "github.com/containerd/containerd/v2/pkg/cap"
19 "github.com/google/go-tpm-tools/cel"
20 "github.com/google/go-tpm-tools/launcher/internal/experiments"
21 "github.com/google/go-tpm-tools/launcher/internal/launchermount"
22 "github.com/google/go-tpm-tools/launcher/internal/logging"
23 "github.com/google/go-tpm-tools/launcher/launcherfile"
24 "github.com/google/go-tpm-tools/verifier"
25 "github.com/google/go-tpm-tools/verifier/util"
26 )
27
28 // MaxInt64 is the maximum value of a signed int64.
29 const MaxInt64 = 9223372036854775807
30
31 // RestartPolicy is the enum for the container restart policy.
32 type RestartPolicy string
33
34 func (p RestartPolicy) isValid() error {
35 switch p {
36 case Always, OnFailure, Never:
37 return nil
38 }
39 return fmt.Errorf("invalid restart policy: %s", p)
40 }
41
42 // Restart Policy enum values.
43 const (
44 Always RestartPolicy = "Always"
45 OnFailure RestartPolicy = "OnFailure"
46 Never RestartPolicy = "Never"
47 // experimentDataFile defines where the experiment sync output data is expected to be.
48 experimentDataFile = "experiment_data"
49 // binaryPath contains the path to the experiments binary.
50 binaryPath = "/usr/share/oem/confidential_space/confidential_space_experiments"
51 )
52
53 // LogRedirectLocation specifies the workload logging redirect location.
54 type LogRedirectLocation string
55
56 func (l LogRedirectLocation) isValid() error {
57 switch l {
58 case Everywhere, CloudLogging, Serial, Nowhere:
59 return nil
60 }
61 return fmt.Errorf("invalid logging redirect location %s, expect one of %s", l,
62 []LogRedirectLocation{Everywhere, CloudLogging, Serial, Nowhere})
63 }
64
65 func (l LogRedirectLocation) enabled() bool {
66 return l != Nowhere
67 }
68
69 // LogRedirectLocation acceptable values.
70 const (
71 Everywhere LogRedirectLocation = "true"
72 CloudLogging LogRedirectLocation = "cloud_logging"
73 Serial LogRedirectLocation = "serial"
74 Nowhere LogRedirectLocation = "false"
75 )
76
77 // Metadata variable names.
78 const (
79 fakeVerifierKey = "test-fake-verifier"
80 imageRefKey = "tee-image-reference"
81 signedImageRepos = "tee-signed-image-repos"
82 restartPolicyKey = "tee-restart-policy"
83 cmdKey = "tee-cmd"
84 envKeyPrefix = "tee-env-"
85 impersonateServiceAccounts = "tee-impersonate-service-accounts"
86 logRedirectKey = "tee-container-log-redirect"
87 memoryMonitoringEnable = "tee-monitoring-memory-enable"
88 monitoringEnable = "tee-monitoring-enable"
89 devShmSizeKey = "tee-dev-shm-size-kb"
90 mountKey = "tee-mount"
91 itaRegion = "ita-region"
92 itaKey = "ita-api-key"
93 addedCaps = "tee-added-capabilities"
94 cgroupNS = "tee-cgroup-ns"
95 gcaServiceEnv = "gca-service-env"
96 installGpuDriver = "tee-install-gpu-driver"
97 disableGcaRefreshKey = "tee-disable-gca-refresh"
98 )
99
100 const (
101 instanceAttributesQuery = "instance/attributes/?recursive=true"
102 )
103
104 var gcaInstances = map[string]string{
105 "prod": "https://confidentialcomputing.googleapis.com",
106 "autopush": "https://autopush-confidentialcomputing.sandbox.googleapis.com",
107 "staging": "https://staging-confidentialcomputing.sandbox.googleapis.com",
108 }
109
110 var errImageRefNotSpecified = fmt.Errorf("%s is not specified in the custom metadata", imageRefKey)
111
112 // EnvVar represent a single environment variable key/value pair.
113 type EnvVar struct {
114 Name string
115 Value string
116 }
117
118 // LaunchSpec contains specification set by the operator who wants to
119 // launch a container.
120 type LaunchSpec struct {
121 Experiments experiments.Experiments
122 FakeVerifierEnabled bool
123
124 // MDS-based values.
125 ImageRef string
126 SignedImageRepos []string
127 RestartPolicy RestartPolicy
128 Cmd []string
129 Envs []EnvVar
130 GcaAddress string
131 ImpersonateServiceAccounts []string
132 ProjectID string
133 Region string
134 Hardened bool
135 MonitoringEnabled MonitoringType
136 LogRedirect LogRedirectLocation
137 Mounts []launchermount.Mount
138 ITAConfig verifier.ITAConfig
139 DevShmSize int64 // DevShmSize is specified in kiB.
140 AddedCapabilities []string
141 CgroupNamespace bool
142 InstallGpuDriver bool
143 DisableGcaRefresh bool
144 }
145
146 // UnmarshalJSON unmarshals an instance attributes list in JSON format from the metadata
147 // server set by an operator to a LaunchSpec.
148 // This method expects experiments to be set on the LaunchSpec before being called.
149 func (s *LaunchSpec) UnmarshalJSON(b []byte) error {
150 var unmarshaledMap map[string]string
151 if err := json.Unmarshal(b, &unmarshaledMap); err != nil {
152 return err
153 }
154
155 if val, ok := unmarshaledMap[fakeVerifierKey]; ok && val != "" {
156 var err error
157 if s.FakeVerifierEnabled, err = strconv.ParseBool(val); err != nil {
158 return fmt.Errorf("invalid value for %v (not a boolean): %w", fakeVerifierKey, err)
159 }
160 }
161
162 if val, ok := unmarshaledMap[installGpuDriver]; ok && val != "" {
163 if boolValue, err := strconv.ParseBool(val); err == nil {
164 s.InstallGpuDriver = boolValue
165 }
166 }
167
168 s.ImageRef = unmarshaledMap[imageRefKey]
169 if s.ImageRef == "" {
170 return errImageRefNotSpecified
171 }
172
173 s.RestartPolicy = RestartPolicy(unmarshaledMap[restartPolicyKey])
174 // Set the default restart policy to "Never" for now.
175 if s.RestartPolicy == "" {
176 s.RestartPolicy = Never
177 }
178 if err := s.RestartPolicy.isValid(); err != nil {
179 return err
180 }
181
182 if val, ok := unmarshaledMap[impersonateServiceAccounts]; ok && val != "" {
183 impersonateAccounts := strings.Split(val, ",")
184 s.ImpersonateServiceAccounts = append(s.ImpersonateServiceAccounts, impersonateAccounts...)
185 }
186
187 if val, ok := unmarshaledMap[signedImageRepos]; ok && val != "" {
188 imageRepos := strings.Split(val, ",")
189 s.SignedImageRepos = append(s.SignedImageRepos, imageRepos...)
190 }
191
192 memVal, memOk := unmarshaledMap[memoryMonitoringEnable]
193 monVal, monOk := unmarshaledMap[monitoringEnable]
194
195 if memOk && monOk {
196 return fmt.Errorf("both %v and %v are specified, only one is permitted", memoryMonitoringEnable, monitoringEnable)
197 } else if memOk {
198 // If value is empty, treat as the default.
199 if memVal == "" {
200 s.MonitoringEnabled = None
201 } else {
202 boolValue, err := strconv.ParseBool(memVal)
203 if err != nil {
204 return fmt.Errorf("invalid value for %v (not a boolean): %v", memoryMonitoringEnable, err)
205 }
206
207 if boolValue {
208 s.MonitoringEnabled = MemoryOnly
209 } else {
210 s.MonitoringEnabled = None
211 }
212 }
213 } else if monOk {
214 // If value is empty, treat as the default.
215 if monVal == "" {
216 s.MonitoringEnabled = None
217 } else {
218 var err error
219 s.MonitoringEnabled, err = toMonitoringType(monVal)
220 if err != nil {
221 return err
222 }
223 }
224 }
225
226 // Populate cmd override.
227 if val, ok := unmarshaledMap[cmdKey]; ok && val != "" {
228 if err := json.Unmarshal([]byte(val), &s.Cmd); err != nil {
229 return err
230 }
231 }
232
233 // Populate all env vars.
234 for k, v := range unmarshaledMap {
235 if strings.HasPrefix(k, envKeyPrefix) {
236 s.Envs = append(s.Envs, EnvVar{strings.TrimPrefix(k, envKeyPrefix), v})
237 }
238 }
239
240 s.LogRedirect = LogRedirectLocation(unmarshaledMap[logRedirectKey])
241 // Default log redirect location is Nowhere ("false").
242 if s.LogRedirect == "" {
243 s.LogRedirect = Nowhere
244 }
245 if err := s.LogRedirect.isValid(); err != nil {
246 return err
247 }
248
249 if err := s.setAttestationServiceVars(unmarshaledMap); err != nil {
250 return err
251 }
252
253 // Populate /dev/shm size override.
254 if val, ok := unmarshaledMap[devShmSizeKey]; ok && val != "" {
255 size, err := strconv.ParseUint(val, 10, 64)
256 if err != nil {
257 return fmt.Errorf("failed to convert %v into uint64, got: %v", devShmSizeKey, val)
258 }
259 s.DevShmSize = int64(size)
260 }
261
262 // Populate mount override.
263 // https://cloud.google.com/compute/docs/disks/set-persistent-device-name-in-linux-vm
264 // https://cloud.google.com/compute/docs/disks/add-local-ssd
265 if val, ok := unmarshaledMap[mountKey]; ok && val != "" {
266 mounts := strings.Split(val, ";")
267 for _, mount := range mounts {
268 specMnt, err := processMount(mount)
269 if err != nil {
270 return err
271 }
272 s.Mounts = append(s.Mounts, specMnt)
273 }
274 }
275
276 if s.Experiments.EnableItaVerifier {
277 itaRegionVal, itaRegionOK := unmarshaledMap[itaRegion]
278 itaKeyVal, itaKeyOK := unmarshaledMap[itaKey]
279
280 // If key and region are both not in the map, do not set up ITA config.
281 if itaRegionOK != itaKeyOK {
282 return fmt.Errorf("ITA fields %s and %s must both be provided and non-empty", itaRegion, itaKey)
283 }
284
285 s.ITAConfig = verifier.ITAConfig{
286 ITARegion: itaRegionVal,
287 ITAKey: itaKeyVal,
288 }
289 }
290
291 // Populate capabilities override.
292 if val, ok := unmarshaledMap[addedCaps]; ok && val != "" {
293 if err := json.Unmarshal([]byte(val), &s.AddedCapabilities); err != nil {
294 return err
295 }
296 }
297
298 // Populate cgroup ns.
299 cgroupSetting, ok := unmarshaledMap[cgroupNS]
300 if ok {
301 cgroupOn, err := strconv.ParseBool(cgroupSetting)
302 if err != nil {
303 return fmt.Errorf("invalid value for %v (not a boolean): %v", cgroupNS, err)
304 }
305 if cgroupOn {
306 s.CgroupNamespace = true
307 }
308 }
309
310 if val, ok := unmarshaledMap[disableGcaRefreshKey]; ok && val != "" {
311 var err error
312 if s.DisableGcaRefresh, err = strconv.ParseBool(val); err != nil {
313 return fmt.Errorf("invalid value for %v (not a boolean): %w", disableGcaRefreshKey, err)
314 }
315 }
316
317 return nil
318 }
319
320 func (s *LaunchSpec) setAttestationServiceVars(unmarshaledMap map[string]string) error {
321 if gcaServiceEnv, ok := unmarshaledMap[gcaServiceEnv]; ok {
322 v, ok := gcaInstances[strings.ToLower(gcaServiceEnv)]
323 if !ok {
324 return fmt.Errorf("the gca service env is not within the allowlist, want %+v, got %s", gcaInstances, gcaServiceEnv)
325 }
326 s.GcaAddress = v
327 }
328
329 return nil
330 }
331
332 // LogFriendly creates a copy of the spec that is safe to log by censoring
333 func (s *LaunchSpec) LogFriendly() LaunchSpec {
334 safeSpec := *s
335 safeSpec.ITAConfig.ITAKey = strings.Repeat("*", len(s.ITAConfig.ITAKey))
336
337 return safeSpec
338 }
339
340 // GetLaunchSpec takes in a metadata server client, reads and parse operator's
341 // input to the GCE instance custom metadata and return a LaunchSpec.
342 // ImageRef (tee-image-reference) is required, will return an error if
343 // ImageRef is not presented in the metadata.
344 func GetLaunchSpec(ctx context.Context, logger logging.Logger, client *metadata.Client) (LaunchSpec, error) {
345 data, err := client.GetWithContext(ctx, instanceAttributesQuery)
346 if err != nil {
347 return LaunchSpec{}, err
348 }
349
350 spec := &LaunchSpec{}
351 spec.Experiments = fetchExperiments(logger)
352 if err := spec.UnmarshalJSON([]byte(data)); err != nil {
353 return LaunchSpec{}, err
354 }
355
356 var errs []error
357 for _, mnt := range spec.Mounts {
358 if err := validateMount(mnt); err != nil {
359 errs = append(errs, err)
360 }
361 }
362 if len(errs) != 0 {
363 return LaunchSpec{}, fmt.Errorf("failed to validate mounts: %v", errors.Join(errs...))
364 }
365
366 if !(spec.Experiments.EnableB200DriverInstallation || spec.Experiments.EnableH100DriverInstallation) && spec.InstallGpuDriver {
367 return LaunchSpec{}, fmt.Errorf("GPU Driver installation is not supported")
368 }
369
370 if err := validateMemorySizeKb(uint64(spec.DevShmSize)); err != nil {
371 return LaunchSpec{}, fmt.Errorf("failed to validate /dev/shm size: %v", err)
372 }
373
374 if err := validateAddedCapsAllowed(spec.AddedCapabilities); err != nil {
375 return LaunchSpec{}, fmt.Errorf("failed to validate added capabilities: %v", err)
376 }
377
378 spec.ProjectID, err = client.ProjectIDWithContext(ctx)
379 if err != nil {
380 return LaunchSpec{}, fmt.Errorf("failed to retrieve projectID from MDS: %v", err)
381 }
382
383 spec.Region, err = util.GetRegion(client)
384 if err != nil {
385 return LaunchSpec{}, err
386 }
387
388 kernelCmd, err := readCmdline()
389 if err != nil {
390 return LaunchSpec{}, err
391 }
392 spec.Hardened = isHardened(kernelCmd)
393
394 return *spec, nil
395 }
396
397 func isHardened(kernelCmd string) bool {
398 for _, arg := range strings.Fields(kernelCmd) {
399 if arg == "confidential-space.hardened=true" {
400 return true
401 }
402 }
403 return false
404 }
405
406 func fetchExperiments(logger logging.Logger) experiments.Experiments {
407 experimentsFile := path.Join(launcherfile.HostTmpPath, experimentDataFile)
408
409 args := fmt.Sprintf("-output=%s", experimentsFile)
410 err := exec.Command(binaryPath, args).Run()
411 if err != nil {
412 logger.Error(fmt.Sprintf("failure during experiment sync: %v\n", err))
413 }
414 e, err := experiments.New(experimentsFile)
415 if err != nil {
416 logger.Error(fmt.Sprintf("failed to read experiment file: %v\n", err))
417 // do not fail if experiment retrieval fails
418 }
419 return e
420 }
421
422 func processMount(singleMount string) (launchermount.Mount, error) {
423 mntConfig := make(map[string]string)
424 var mntType string
425 mountOpts := strings.Split(singleMount, ",")
426 for _, mountOpt := range mountOpts {
427 name, val, err := cel.ParseEnvVar(mountOpt)
428 if err != nil {
429 return nil, fmt.Errorf("failed to parse mount option: %w", err)
430 }
431 switch name {
432 case launchermount.TypeKey:
433 mntType = val
434 case launchermount.SourceKey:
435 case launchermount.DestinationKey:
436 case launchermount.SizeKey:
437 default:
438 return nil, fmt.Errorf("found unknown mount option: %v, expect keys of %v", mountOpt, launchermount.AllMountKeys)
439 }
440 mntConfig[name] = val
441 }
442
443 switch mntType {
444 case launchermount.TypeTmpfs:
445 return launchermount.CreateTmpfsMount(mntConfig)
446 default:
447 return nil, fmt.Errorf("found unknown or unspecified mount type: %v, expect one of types [%v]", mountOpts, launchermount.TypeTmpfs)
448 }
449 }
450
451 func validateMount(mnt launchermount.Mount) error {
452 switch v := mnt.(type) {
453 case launchermount.TmpfsMount:
454 return validateMemorySizeKb(v.Size / 1024)
455 default:
456 return fmt.Errorf("got unknown mount type: %T", v)
457 }
458 }
459
460 // Ensures that system free memory is larger than the specified memory size.
461 func validateMemorySizeKb(memSize uint64) error {
462 freeMem, err := getLinuxFreeMem()
463 if err != nil {
464 return fmt.Errorf("failed to get free memory: %v", err)
465 }
466 if memSize > freeMem {
467 return fmt.Errorf("got a /dev/shm size (%v) larger than free memory (%v) kB", memSize, freeMem)
468 }
469 if memSize > MaxInt64 {
470 return fmt.Errorf("got a size greater than max int64: %v", memSize)
471 }
472 return nil
473 }
474
475 func getLinuxFreeMem() (uint64, error) {
476 meminfo, err := os.ReadFile("/proc/meminfo")
477 if err != nil {
478 return 0, fmt.Errorf("failed to read /proc/meminfo: %w", err)
479 }
480 for _, memtype := range strings.Split(string(meminfo), "\n") {
481 if !strings.Contains(memtype, "MemFree") {
482 continue
483 }
484 split := strings.Fields(memtype)
485 if len(split) != 3 {
486 return 0, fmt.Errorf("found invalid MemInfo entry: got: %v, expected format: MemFree: <amount> kB", memtype)
487 }
488 if split[2] != "kB" {
489 return 0, fmt.Errorf("found invalid MemInfo entry: got: %v, expected format: MemFree: <amount> kB", memtype)
490 }
491 freeMem, err := strconv.ParseUint(split[1], 10, 64)
492 if err != nil {
493 return 0, fmt.Errorf("failed to convert MemFree to uint64: %v", memtype)
494 }
495 return freeMem, nil
496 }
497 return 0, fmt.Errorf("failed to find MemFree in /proc/meminfo: %v", string(meminfo))
498 }
499
500 func readCmdline() (string, error) {
501 kernelCmd, err := os.ReadFile("/proc/cmdline")
502 if err != nil {
503 return "", err
504 }
505 return string(kernelCmd), nil
506 }
507
508 func validateAddedCapsAllowed(addedCaps []string) error {
509 caps, err := getCurrCaps()
510 if err != nil {
511 return fmt.Errorf("failed to fetch current capabilities: %v", err)
512 }
513 var notInCurr []string
514 for _, addedCap := range addedCaps {
515 if _, ok := caps[addedCap]; !ok {
516 notInCurr = append(notInCurr, addedCap)
517 }
518 }
519 if len(notInCurr) != 0 {
520 return fmt.Errorf("received added capabilities (%v) not allowed by current capabilities", notInCurr)
521
522 }
523 return nil
524 }
525
526 func getCurrCaps() (map[string]bool, error) {
527 caps, err := cap.Current()
528 if err != nil {
529 return nil, err
530 }
531 capsMap := make(map[string]bool, len(caps))
532 for _, cap := range caps {
533 capsMap[cap] = true
534 }
535 return capsMap, nil
536 }