Debian Codegraph @

examples/docs_examples/linear_regression.py

raw
1 """
2 Example of using FFCV to speed up large scale linear regression.
3 For tutorial, see https://docs.ffcv.io/ffcv_examples/linear_regression.html.
4
5 """
6 from tqdm import tqdm
7 import time
8 import numpy as np
9 import pickle as pkl
10 import torch as ch
11 from torch.utils.data import TensorDataset, DataLoader
12 from ffcv.fields import NDArrayField, FloatField
13 from ffcv.fields.basics import FloatDecoder
14 from ffcv.fields.decoders import NDArrayDecoder
15 from ffcv.loader import Loader, OrderOption
16 from ffcv.writer import DatasetWriter
17 from ffcv.transforms import ToTensor, ToDevice, Squeeze
18 import os
19
20 # 1,000,000 inputs each of dimension 10,000 = 40GB of data
21 N, D = 1000000, 10000
22 USE_FFCV = True
23 if not os.path.exists('/tmp/linreg_data.pkl'):
24 X = np.random.rand(N, D).astype('float32')
25 # Ground-truth vector
26 W, b = np.random.rand(D).astype('float32'), np.random.rand()
27 # Response variables
28 Y = X @ W + b + np.random.randn(N).astype('float32')
29 pkl.dump((X, W, b, Y), open('/tmp/linreg_data.pkl', 'wb'))
30 elif not USE_FFCV:
31 print('Loading from disk...')
32 X, W, b, Y = pkl.load(open('/tmp/linreg_data.pkl', 'rb'))
33
34 if USE_FFCV and not os.path.exists('/tmp/linreg_data.beton'):
35 X, W, b, Y = pkl.load(open('/tmp/linreg_data.pkl', 'rb'))
36 class LinearRegressionDataset:
37 def __getitem__(self, idx):
38 return (X[idx], np.array(Y[idx]).astype('float32'))
39
40 def __len__(self):
41 return len(X)
42
43 writer = DatasetWriter('/tmp/linreg_data.beton', {
44 'covariate': NDArrayField(shape=(D,), dtype=np.dtype('float32')),
45 'label': NDArrayField(shape=(1,), dtype=np.dtype('float32')),
46 }, num_workers=16)
47
48 writer.from_indexed_dataset(LinearRegressionDataset())
49 else:
50 print('FFCV file already written')
51
52
53 ### PART 2: actual regression
54
55 if not USE_FFCV:
56 dataset = TensorDataset(ch.tensor(X), ch.tensor(Y))
57 train_loader = DataLoader(dataset, batch_size=2048, num_workers=8, shuffle=True)
58 else:
59 train_loader = Loader('/tmp/linreg_data.beton', batch_size=2048,
60 num_workers=8, order=OrderOption.QUASI_RANDOM, os_cache=False,
61 pipelines={
62 'covariate': [NDArrayDecoder(), ToTensor(), ToDevice(ch.device('cuda:0'))],
63 'label': [NDArrayDecoder(), ToTensor(), Squeeze(), ToDevice(ch.device('cuda:0'))]
64 })
65
66 # Calculate data mean and variance for normalization
67 def calculate_stats(loader, N):
68 mean, stdev = 0., 0.
69 for x_batch, _ in tqdm(loader):
70 mean += x_batch.sum(0) / N
71 stdev += x_batch.pow(2).sum(0) / N
72 return mean, ch.sqrt(stdev - mean.pow(2))
73
74 mean, stdev = calculate_stats(train_loader, N)
75 mean, stdev = mean.cuda(), stdev.cuda()
76 w_est, b_est = ch.zeros(D).cuda(), ch.zeros(1).cuda() # Initial guess for W
77 num_epochs = 10 # Number of full passes over the data to do
78
79 lr = 5e-2
80 for _ in range(num_epochs):
81 total_loss, num_examples = 0., 0.
82 start_time = time.time()
83 for (x_batch, y_batch) in tqdm(train_loader):
84 if not USE_FFCV:
85 x_batch = x_batch.cuda()
86 y_batch = y_batch.cuda()
87 # Normalize the data for stability
88 x_batch = (x_batch - mean) / stdev
89 residual = x_batch @ w_est + b_est - y_batch
90 # Gradients
91 w_grad = x_batch.T @ residual / x_batch.shape[0]
92 b_grad = ch.mean(residual, dim=0)
93 w_est = w_est - lr * w_grad
94 b_est = b_est - lr * b_grad
95 total_loss += residual.pow(2).sum()
96 num_examples += x_batch.shape[0]
97 print('Epoch time:', time.time() - start_time)
98 print(f'Average loss: {total_loss / num_examples:.3f}')