| 1 |
// Package gpu provides the devices information for cos-gpu-installer |
| 2 |
package gpu |
| 3 |
|
| 4 |
import ( |
| 5 |
"context" |
| 6 |
"crypto/sha256" |
| 7 |
"encoding/hex" |
| 8 |
"fmt" |
| 9 |
"os" |
| 10 |
"os/exec" |
| 11 |
"path" |
| 12 |
"strings" |
| 13 |
|
| 14 |
"cos.googlesource.com/cos/tools.git/src/cmd/cos_gpu_installer/deviceinfo" |
| 15 |
"github.com/containerd/containerd" |
| 16 |
"github.com/containerd/containerd/cio" |
| 17 |
"github.com/containerd/containerd/namespaces" |
| 18 |
"github.com/containerd/containerd/oci" |
| 19 |
"github.com/google/go-tpm-tools/launcher/internal/logging" |
| 20 |
"github.com/google/go-tpm-tools/launcher/spec" |
| 21 |
"github.com/google/go-tpm-tools/proto/attest" |
| 22 |
"github.com/opencontainers/runtime-spec/specs-go" |
| 23 |
) |
| 24 |
|
| 25 |
const ( |
| 26 |
installerContainerID = "tee-gpu-driver-installer-container" |
| 27 |
installerSnapshotID = "tee-gpu-driver-installer-snapshot" |
| 28 |
) |
| 29 |
|
| 30 |
// NvidiaSmiCmdOutput defines a function type for executing an NVIDIA SMI command |
| 31 |
// and returning the raw byte output along with any error. |
| 32 |
type NvidiaSmiCmdOutput func() ([]byte, error) |
| 33 |
|
| 34 |
// DriverInstaller contains information about the GPU driver installer settings |
| 35 |
type DriverInstaller struct { |
| 36 |
cdClient *containerd.Client |
| 37 |
launchSpec spec.LaunchSpec |
| 38 |
logger logging.Logger |
| 39 |
} |
| 40 |
|
| 41 |
// NewDriverInstaller instantiates an object of driver installer |
| 42 |
func NewDriverInstaller(cdClient *containerd.Client, launchSpec spec.LaunchSpec, logger logging.Logger) *DriverInstaller { |
| 43 |
return &DriverInstaller{ |
| 44 |
cdClient: cdClient, |
| 45 |
launchSpec: launchSpec, |
| 46 |
logger: logger, |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
// InstallGPUDrivers installs the GPU driver on host machine using the cos-gpu-installer container. |
| 51 |
// This function performs the same steps specified in this README file: |
| 52 |
// https://pkg.go.dev/cos.googlesource.com/cos/[email protected]/src/cmd/cos_gpu_installer#section-readme |
| 53 |
// README specifies docker command where this function uses containerd for launching and managing the GPU driver installer container. |
| 54 |
func (di *DriverInstaller) InstallGPUDrivers(ctx context.Context) error { |
| 55 |
err := remountAsExecutable(InstallationHostDir) |
| 56 |
if err != nil { |
| 57 |
return fmt.Errorf("failed to remount the installation directory: %v", err) |
| 58 |
} |
| 59 |
|
| 60 |
gpuType, err := deviceinfo.GetGPUTypeInfo() |
| 61 |
if err != nil { |
| 62 |
return fmt.Errorf("failed to get the GPU type info: %v", err) |
| 63 |
} |
| 64 |
|
| 65 |
switch gpuType { |
| 66 |
case deviceinfo.B200: |
| 67 |
case deviceinfo.H100: |
| 68 |
default: |
| 69 |
return fmt.Errorf("unsupported GPU type %s for Confidential Computing (only supported for H100[a3-highgpu-1g] and B200[a4-highgpu-8g])", gpuType.String()) |
| 70 |
} |
| 71 |
|
| 72 |
ctx = namespaces.WithNamespace(ctx, namespaces.Default) |
| 73 |
di.logger.Info(fmt.Sprintf("COS GPU installer version : %s", InstallerImageRef)) |
| 74 |
image, err := di.cdClient.Pull(ctx, InstallerImageRef, containerd.WithPullUnpack) |
| 75 |
if err != nil { |
| 76 |
return fmt.Errorf("failed to pull installer image: %v", err) |
| 77 |
} |
| 78 |
if err := verifyInstallerImageDigest(image, InstallerDigest); err != nil { |
| 79 |
return err |
| 80 |
} |
| 81 |
|
| 82 |
mounts := []specs.Mount{ |
| 83 |
{ |
| 84 |
Type: "volume", |
| 85 |
Source: "/dev", |
| 86 |
Destination: "/dev", |
| 87 |
Options: []string{"rbind", "rw"}, |
| 88 |
}, { |
| 89 |
Type: "volume", |
| 90 |
Source: "/", |
| 91 |
Destination: "/root", |
| 92 |
Options: []string{"rbind", "rw"}, |
| 93 |
}, |
| 94 |
} |
| 95 |
|
| 96 |
hostname, err := os.Hostname() |
| 97 |
if err != nil { |
| 98 |
di.logger.Error(fmt.Sprintf("cannot get hostname: %v", err)) |
| 99 |
} |
| 100 |
|
| 101 |
container, err := di.cdClient.NewContainer( |
| 102 |
ctx, |
| 103 |
installerContainerID, |
| 104 |
containerd.WithImage(image), |
| 105 |
containerd.WithNewSnapshot(installerSnapshotID, image), |
| 106 |
containerd.WithNewSpec(oci.WithImageConfig(image), |
| 107 |
oci.WithPrivileged, |
| 108 |
// To support confidential GPUs, the nvidia-persistenced process should be started before the GPU driver verification step. |
| 109 |
// It would not be possible to start the nvidia-persistenced process amidst GPU driver installation flow via cos_gpu_installer. |
| 110 |
// For this reason, the GPU driver installation need to be triggered with --skip-nvidia-smi flag to skip the GPU driver verification step. |
| 111 |
oci.WithProcessArgs("/cos-gpu-installer", "install", |
| 112 |
fmt.Sprintf("-version=%s", NvDriverVer590_48_01), |
| 113 |
fmt.Sprintf("-host-dir=%s", InstallationHostDir), |
| 114 |
"--no-verify"), |
| 115 |
oci.WithAllDevicesAllowed, |
| 116 |
oci.WithHostDevices, |
| 117 |
oci.WithMounts(mounts), |
| 118 |
oci.WithHostNamespace(specs.NetworkNamespace), |
| 119 |
oci.WithHostNamespace(specs.PIDNamespace), |
| 120 |
oci.WithHostHostsFile, |
| 121 |
oci.WithHostResolvconf, |
| 122 |
oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}))) |
| 123 |
if err != nil { |
| 124 |
return fmt.Errorf("failed to create GPU driver installer container: %v", err) |
| 125 |
} |
| 126 |
defer container.Delete(ctx, containerd.WithSnapshotCleanup) |
| 127 |
|
| 128 |
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio)) |
| 129 |
if err != nil { |
| 130 |
return fmt.Errorf("failed to create GPU driver installation task: %v", err) |
| 131 |
} |
| 132 |
defer task.Delete(ctx) |
| 133 |
|
| 134 |
statusC, err := task.Wait(ctx) |
| 135 |
if err != nil { |
| 136 |
return fmt.Errorf("failed to wait for GPU driver installation task: %v", err) |
| 137 |
} |
| 138 |
|
| 139 |
if err := task.Start(ctx); err != nil { |
| 140 |
return fmt.Errorf("failed to start GPU driver installation task: %v", err) |
| 141 |
} |
| 142 |
|
| 143 |
status := <-statusC |
| 144 |
code, _, _ := status.Result() |
| 145 |
|
| 146 |
if code != 0 { |
| 147 |
di.logger.Error(fmt.Sprintf("GPU driver installation task ended and returned non-zero status code %d", code)) |
| 148 |
return fmt.Errorf("GPU driver installation task ended with non-zero status code %d", code) |
| 149 |
} |
| 150 |
|
| 151 |
if err = verifyDriverDigest(path.Join(InstallationHostDir, NvDriverVer590_48_01Runfile), NvDriverVer590_48_01Digest); err != nil { |
| 152 |
return fmt.Errorf("failed to verify GPU driver digest: %v", err) |
| 153 |
} |
| 154 |
|
| 155 |
cmds := [][]string{ |
| 156 |
{"/sbin/modprobe", "drm_kms_helper"}, |
| 157 |
{"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia.ko"}, |
| 158 |
{"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-uvm.ko"}, |
| 159 |
{"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-modeset.ko"}, |
| 160 |
{"/sbin/insmod", "/var/lib/nvidia/drivers/nvidia-drm.ko"}, |
| 161 |
} |
| 162 |
|
| 163 |
for _, cmdArgs := range cmds { |
| 164 |
cmd := exec.Command("sudo", cmdArgs...) |
| 165 |
if out, err := cmd.CombinedOutput(); err != nil { |
| 166 |
di.logger.Info(fmt.Sprintf("failed to run command %v: %v, output: %s", cmdArgs, err, string(out))) |
| 167 |
} |
| 168 |
} |
| 169 |
|
| 170 |
if err = launchNvidiaPersistencedProcess(di.logger); err != nil { |
| 171 |
return fmt.Errorf("failed to start nvidia-persistenced process: %v", err) |
| 172 |
} |
| 173 |
|
| 174 |
nvidiaSmiVerifyCmd := NvidiaSmiOutputFunc() |
| 175 |
if err = verifyDriverInstallation(nvidiaSmiVerifyCmd); err != nil { |
| 176 |
return fmt.Errorf("failed to verify GPU driver installation: %v", err) |
| 177 |
} |
| 178 |
|
| 179 |
if gpuType == deviceinfo.B200 { |
| 180 |
topologyVerificationCmd := NvidiaSmiOutputFunc("topo", "-m") |
| 181 |
if err = verifyDriverInstallation(topologyVerificationCmd); err != nil { |
| 182 |
return fmt.Errorf("failed to verify Multi GPU topology: %v", err) |
| 183 |
} |
| 184 |
} |
| 185 |
|
| 186 |
di.logger.Info("GPU driver installation completed successfully") |
| 187 |
return nil |
| 188 |
} |
| 189 |
|
| 190 |
func verifyInstallerImageDigest(image containerd.Image, referenceDigest string) error { |
| 191 |
installerDigest := image.Target().Digest.String() |
| 192 |
if installerDigest != referenceDigest { |
| 193 |
return fmt.Errorf("cos_gpu_installer image digest verification failed - expected : %s, actual : %s", referenceDigest, installerDigest) |
| 194 |
} |
| 195 |
return nil |
| 196 |
} |
| 197 |
|
| 198 |
func verifyDriverDigest(driverFilePath, referenceDigest string) error { |
| 199 |
contentBytes, err := os.ReadFile(driverFilePath) |
| 200 |
if err != nil { |
| 201 |
return fmt.Errorf("failed to read the file %s, got error %v", driverFilePath, err) |
| 202 |
} |
| 203 |
checksum := sha256.Sum256(contentBytes) |
| 204 |
checksumHex := hex.EncodeToString(checksum[:]) |
| 205 |
if checksumHex != referenceDigest { |
| 206 |
return fmt.Errorf("GPU driver digest verification failed - expected : %s, got : %s", referenceDigest, checksumHex) |
| 207 |
} |
| 208 |
return nil |
| 209 |
} |
| 210 |
|
| 211 |
func remountAsExecutable(dir string) error { |
| 212 |
if err := os.MkdirAll(dir, 0755); err != nil { |
| 213 |
return fmt.Errorf("failed to create dir %q: %v", dir, err) |
| 214 |
} |
| 215 |
if err := exec.Command("mount", "--bind", dir, dir).Run(); err != nil { |
| 216 |
return fmt.Errorf("failed to create bind mount at %q: %v", dir, err) |
| 217 |
} |
| 218 |
if err := exec.Command("mount", "-o", "remount,exec", dir).Run(); err != nil { |
| 219 |
return fmt.Errorf("failed to remount %q: %v", dir, err) |
| 220 |
} |
| 221 |
return nil |
| 222 |
} |
| 223 |
|
| 224 |
func verifyDriverInstallation(nvidiaSmiVerifyCmd NvidiaSmiCmdOutput) error { |
| 225 |
if _, err := nvidiaSmiVerifyCmd(); err != nil { |
| 226 |
return fmt.Errorf("failed to verify GPU driver installation : %v", err) |
| 227 |
} |
| 228 |
return nil |
| 229 |
} |
| 230 |
|
| 231 |
func setGPUStateToReady(nvidiaSmiSetGPUStateCmd NvidiaSmiCmdOutput) error { |
| 232 |
if _, err := nvidiaSmiSetGPUStateCmd(); err != nil { |
| 233 |
return fmt.Errorf("failed to set the GPU state to ready: %v", err) |
| 234 |
} |
| 235 |
return nil |
| 236 |
} |
| 237 |
|
| 238 |
// QueryCCMode executes nvidia-smi to determine the current Confidential Computing (CC) mode status of the GPU. |
| 239 |
// If DEVTOOLS mode is enabled, it would override CC mode as DEVTOOLS. DEVTOOLS mode would be enabled only when CC mode is ON. |
| 240 |
func QueryCCMode(ccModeCmd, devToolsCmd NvidiaSmiCmdOutput) (attest.GPUDeviceCCMode, error) { |
| 241 |
ccMode := attest.GPUDeviceCCMode_UNSET |
| 242 |
ccModeOutput, err := ccModeCmd() |
| 243 |
if err != nil { |
| 244 |
return attest.GPUDeviceCCMode_UNSET, err |
| 245 |
} |
| 246 |
|
| 247 |
devToolsOutput, err := devToolsCmd() |
| 248 |
if err != nil { |
| 249 |
return attest.GPUDeviceCCMode_UNSET, err |
| 250 |
} |
| 251 |
|
| 252 |
if strings.Contains(string(ccModeOutput), "CC status: ON") { |
| 253 |
ccMode = attest.GPUDeviceCCMode_ON |
| 254 |
} else if strings.Contains(string(ccModeOutput), "CC status: OFF") { |
| 255 |
ccMode = attest.GPUDeviceCCMode_OFF |
| 256 |
} |
| 257 |
|
| 258 |
if ccMode == attest.GPUDeviceCCMode_ON && strings.Contains(string(devToolsOutput), "DevTools Mode: ON") { |
| 259 |
ccMode = attest.GPUDeviceCCMode_DEVTOOLS |
| 260 |
} |
| 261 |
|
| 262 |
return ccMode, nil |
| 263 |
} |
| 264 |
|
| 265 |
func launchNvidiaPersistencedProcess(logger logging.Logger) error { |
| 266 |
nvidiaPersistencedCmd := fmt.Sprintf("%s/bin/nvidia-persistenced", InstallationHostDir) |
| 267 |
logger.Info("Starting nvidia-persistenced process") |
| 268 |
if err := exec.Command(nvidiaPersistencedCmd).Run(); err != nil { |
| 269 |
return fmt.Errorf("failed to launch nvidia-persistenced daemon: %v", err) |
| 270 |
} |
| 271 |
logger.Info("nvidia-persistenced daemon successfully started") |
| 272 |
return nil |
| 273 |
} |
| 274 |
|
| 275 |
// NvidiaSmiOutputFunc returns a function which executes the nvidia-smi command with the given arguments |
| 276 |
// and returns the raw byte output and any error. |
| 277 |
func NvidiaSmiOutputFunc(args ...string) NvidiaSmiCmdOutput { |
| 278 |
cmd := fmt.Sprintf("%s/bin/nvidia-smi", InstallationHostDir) |
| 279 |
return func() ([]byte, error) { return exec.Command(cmd, args...).Output() } |
| 280 |
} |