Debian Codegraph @

launcher/internal/gpu/driverinstaller.go

raw
1 // Package gpu provides the devices information for cos-gpu-installer
2 package gpu
3
4 import (
5 "context"
6 "crypto/sha256"
7 "encoding/hex"
8 "fmt"
9 "os"
10 "os/exec"
11 "path"
12 "strings"
13
14 "cos.googlesource.com/cos/tools.git/src/cmd/cos_gpu_installer/deviceinfo"
15 "github.com/containerd/containerd"
16 "github.com/containerd/containerd/cio"
17 "github.com/containerd/containerd/namespaces"
18 "github.com/containerd/containerd/oci"
19 "github.com/google/go-tpm-tools/launcher/internal/logging"
20 "github.com/google/go-tpm-tools/launcher/spec"
21 "github.com/google/go-tpm-tools/proto/attest"
22 "github.com/opencontainers/runtime-spec/specs-go"
23 )
24
25 const (
26 installerContainerID = "tee-gpu-driver-installer-container"
27 installerSnapshotID = "tee-gpu-driver-installer-snapshot"
28 )
29
30 // NvidiaSmiCmdOutput defines a function type for executing an NVIDIA SMI command
31 // and returning the raw byte output along with any error.
32 type NvidiaSmiCmdOutput func() ([]byte, error)
33
34 // DriverInstaller contains information about the GPU driver installer settings
35 type DriverInstaller struct {
36 cdClient *containerd.Client
37 launchSpec spec.LaunchSpec
38 logger logging.Logger
39 }
40
41 // NewDriverInstaller instantiates an object of driver installer
42 func NewDriverInstaller(cdClient *containerd.Client, launchSpec spec.LaunchSpec, logger logging.Logger) *DriverInstaller {
43 return &DriverInstaller{
44 cdClient: cdClient,
45 launchSpec: launchSpec,
46 logger: logger,
47 }
48 }
49
50 // InstallGPUDrivers installs the GPU driver on host machine using the cos-gpu-installer container.
51 // This function performs the same steps specified in this README file:
52 // https://pkg.go.dev/cos.googlesource.com/cos/[email protected]/src/cmd/cos_gpu_installer#section-readme
53 // README specifies docker command where this function uses containerd for launching and managing the GPU driver installer container.
54 func (di *DriverInstaller) InstallGPUDrivers(ctx context.Context) error {
55 err := remountAsExecutable(InstallationHostDir)
56 if err != nil {
57 return fmt.Errorf("failed to remount the installation directory: %v", err)
58 }
59
60 gpuType, err := deviceinfo.GetGPUTypeInfo()
61 if err != nil {
62 return fmt.Errorf("failed to get the GPU type info: %v", err)
63 }
64
65 switch gpuType {
66 case deviceinfo.B200:
67 case deviceinfo.H100:
68 default:
69 return fmt.Errorf("unsupported GPU type %s for Confidential Computing (only supported for H100[a3-highgpu-1g] and B200[a4-highgpu-8g])", gpuType.String())
70 }
71
72 ctx = namespaces.WithNamespace(ctx, namespaces.Default)
73 di.logger.Info(fmt.Sprintf("COS GPU installer version : %s", InstallerImageRef))
74 image, err := di.cdClient.Pull(ctx, InstallerImageRef, containerd.WithPullUnpack)
75 if err != nil {
76 return fmt.Errorf("failed to pull installer image: %v", err)
77 }
78 if err := verifyInstallerImageDigest(image, InstallerDigest); err != nil {
79 return err
80 }
81
82 mounts := []specs.Mount{
83 {
84 Type: "volume",
85 Source: "/dev",
86 Destination: "/dev",
87 Options: []string{"rbind", "rw"},
88 }, {
89 Type: "volume",
90 Source: "/",
91 Destination: "/root",
92 Options: []string{"rbind", "rw"},
93 },
94 }
95
96 hostname, err := os.Hostname()
97 if err != nil {
98 di.logger.Error(fmt.Sprintf("cannot get hostname: %v", err))
99 }
100
101 container, err := di.cdClient.NewContainer(
102 ctx,
103 installerContainerID,
104 containerd.WithImage(image),
105 containerd.WithNewSnapshot(installerSnapshotID, image),
106 containerd.WithNewSpec(oci.WithImageConfig(image),
107 oci.WithPrivileged,
108 // To support confidential GPUs, the nvidia-persistenced process should be started before the GPU driver verification step.
109 // It would not be possible to start the nvidia-persistenced process amidst GPU driver installation flow via cos_gpu_installer.
110 // For this reason, the GPU driver installation need to be triggered with --skip-nvidia-smi flag to skip the GPU driver verification step.
111 oci.WithProcessArgs("/cos-gpu-installer", "install",
112 fmt.Sprintf("-version=%s", NvDriverVer590_48_01),
113 fmt.Sprintf("-host-dir=%s", InstallationHostDir),
114 "--no-verify"),
115 oci.WithAllDevicesAllowed,
116 oci.WithHostDevices,
117 oci.WithMounts(mounts),
118 oci.WithHostNamespace(specs.NetworkNamespace),
119 oci.WithHostNamespace(specs.PIDNamespace),
120 oci.WithHostHostsFile,
121 oci.WithHostResolvconf,
122 oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)})))
123 if err != nil {
124 return fmt.Errorf("failed to create GPU driver installer container: %v", err)
125 }
126 defer container.Delete(ctx, containerd.WithSnapshotCleanup)
127
128 task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio))
129 if err != nil {
130 return fmt.Errorf("failed to create GPU driver installation task: %v", err)
131 }
132 defer task.Delete(ctx)
133
134 statusC, err := task.Wait(ctx)
135 if err != nil {
136 return fmt.Errorf("failed to wait for GPU driver installation task: %v", err)
137 }
138
139 if err := task.Start(ctx); err != nil {
140 return fmt.Errorf("failed to start GPU driver installation task: %v", err)
141 }
142
143 status := <-statusC
144 code, _, _ := status.Result()
145
146 if code != 0 {
147 di.logger.Error(fmt.Sprintf("GPU driver installation task ended and returned non-zero status code %d", code))
148 return fmt.Errorf("GPU driver installation task ended with non-zero status code %d", code)
149 }
150
151 if err = verifyDriverDigest(path.Join(InstallationHostDir, NvDriverVer590_48_01Runfile), NvDriverVer590_48_01Digest); err != nil {
152 return fmt.Errorf("failed to verify GPU driver digest: %v", err)
153 }
154
155 cmds := [][]string{
156 {"/sbin/modprobe", "drm_kms_helper"},
157 {"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia.ko"},
158 {"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-uvm.ko"},
159 {"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-modeset.ko"},
160 {"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-drm.ko"},
161 }
162
163 for _, cmdArgs := range cmds {
164 cmd := exec.Command("sudo", cmdArgs...)
165 if out, err := cmd.CombinedOutput(); err != nil {
166 di.logger.Info(fmt.Sprintf("failed to run command %v: %v, output: %s", cmdArgs, err, string(out)))
167 }
168 }
169
170 if err = launchNvidiaPersistencedProcess(di.logger); err != nil {
171 return fmt.Errorf("failed to start nvidia-persistenced process: %v", err)
172 }
173
174 nvidiaSmiVerifyCmd := NvidiaSmiOutputFunc()
175 if err = verifyDriverInstallation(nvidiaSmiVerifyCmd); err != nil {
176 return fmt.Errorf("failed to verify GPU driver installation: %v", err)
177 }
178
179 if gpuType == deviceinfo.B200 {
180 topologyVerificationCmd := NvidiaSmiOutputFunc("topo", "-m")
181 if err = verifyDriverInstallation(topologyVerificationCmd); err != nil {
182 return fmt.Errorf("failed to verify Multi GPU topology: %v", err)
183 }
184 }
185
186 di.logger.Info("GPU driver installation completed successfully")
187 return nil
188 }
189
190 func verifyInstallerImageDigest(image containerd.Image, referenceDigest string) error {
191 installerDigest := image.Target().Digest.String()
192 if installerDigest != referenceDigest {
193 return fmt.Errorf("cos_gpu_installer image digest verification failed - expected : %s, actual : %s", referenceDigest, installerDigest)
194 }
195 return nil
196 }
197
198 func verifyDriverDigest(driverFilePath, referenceDigest string) error {
199 contentBytes, err := os.ReadFile(driverFilePath)
200 if err != nil {
201 return fmt.Errorf("failed to read the file %s, got error %v", driverFilePath, err)
202 }
203 checksum := sha256.Sum256(contentBytes)
204 checksumHex := hex.EncodeToString(checksum[:])
205 if checksumHex != referenceDigest {
206 return fmt.Errorf("GPU driver digest verification failed - expected : %s, got : %s", referenceDigest, checksumHex)
207 }
208 return nil
209 }
210
211 func remountAsExecutable(dir string) error {
212 if err := os.MkdirAll(dir, 0755); err != nil {
213 return fmt.Errorf("failed to create dir %q: %v", dir, err)
214 }
215 if err := exec.Command("mount", "--bind", dir, dir).Run(); err != nil {
216 return fmt.Errorf("failed to create bind mount at %q: %v", dir, err)
217 }
218 if err := exec.Command("mount", "-o", "remount,exec", dir).Run(); err != nil {
219 return fmt.Errorf("failed to remount %q: %v", dir, err)
220 }
221 return nil
222 }
223
224 func verifyDriverInstallation(nvidiaSmiVerifyCmd NvidiaSmiCmdOutput) error {
225 if _, err := nvidiaSmiVerifyCmd(); err != nil {
226 return fmt.Errorf("failed to verify GPU driver installation : %v", err)
227 }
228 return nil
229 }
230
231 func setGPUStateToReady(nvidiaSmiSetGPUStateCmd NvidiaSmiCmdOutput) error {
232 if _, err := nvidiaSmiSetGPUStateCmd(); err != nil {
233 return fmt.Errorf("failed to set the GPU state to ready: %v", err)
234 }
235 return nil
236 }
237
238 // QueryCCMode executes nvidia-smi to determine the current Confidential Computing (CC) mode status of the GPU.
239 // If DEVTOOLS mode is enabled, it would override CC mode as DEVTOOLS. DEVTOOLS mode would be enabled only when CC mode is ON.
240 func QueryCCMode(ccModeCmd, devToolsCmd NvidiaSmiCmdOutput) (attest.GPUDeviceCCMode, error) {
241 ccMode := attest.GPUDeviceCCMode_UNSET
242 ccModeOutput, err := ccModeCmd()
243 if err != nil {
244 return attest.GPUDeviceCCMode_UNSET, err
245 }
246
247 devToolsOutput, err := devToolsCmd()
248 if err != nil {
249 return attest.GPUDeviceCCMode_UNSET, err
250 }
251
252 if strings.Contains(string(ccModeOutput), "CC status: ON") {
253 ccMode = attest.GPUDeviceCCMode_ON
254 } else if strings.Contains(string(ccModeOutput), "CC status: OFF") {
255 ccMode = attest.GPUDeviceCCMode_OFF
256 }
257
258 if ccMode == attest.GPUDeviceCCMode_ON && strings.Contains(string(devToolsOutput), "DevTools Mode: ON") {
259 ccMode = attest.GPUDeviceCCMode_DEVTOOLS
260 }
261
262 return ccMode, nil
263 }
264
265 func launchNvidiaPersistencedProcess(logger logging.Logger) error {
266 nvidiaPersistencedCmd := fmt.Sprintf("%s/bin/nvidia-persistenced", InstallationHostDir)
267 logger.Info("Starting nvidia-persistenced process")
268 if err := exec.Command(nvidiaPersistencedCmd).Run(); err != nil {
269 return fmt.Errorf("failed to launch nvidia-persistenced daemon: %v", err)
270 }
271 logger.Info("nvidia-persistenced daemon successfully started")
272 return nil
273 }
274
275 // NvidiaSmiOutputFunc returns a function which executes the nvidia-smi command with the given arguments
276 // and returns the raw byte output and any error.
277 func NvidiaSmiOutputFunc(args ...string) NvidiaSmiCmdOutput {
278 cmd := fmt.Sprintf("%s/bin/nvidia-smi", InstallationHostDir)
279 return func() ([]byte, error) { return exec.Command(cmd, args...).Output() }
280 }