| 1 |
""" |
| 2 |
Example of using FFCV to speed up large scale linear regression. |
| 3 |
For tutorial, see https://docs.ffcv.io/ffcv_examples/linear_regression.html. |
| 4 |
|
| 5 |
""" |
| 6 |
from tqdm import tqdm |
| 7 |
import time |
| 8 |
import numpy as np |
| 9 |
import pickle as pkl |
| 10 |
import torch as ch |
| 11 |
from torch.utils.data import TensorDataset, DataLoader |
| 12 |
from ffcv.fields import NDArrayField, FloatField |
| 13 |
from ffcv.fields.basics import FloatDecoder |
| 14 |
from ffcv.fields.decoders import NDArrayDecoder |
| 15 |
from ffcv.loader import Loader, OrderOption |
| 16 |
from ffcv.writer import DatasetWriter |
| 17 |
from ffcv.transforms import ToTensor, ToDevice, Squeeze |
| 18 |
import os |
| 19 |
|
| 20 |
# 1,000,000 inputs each of dimension 10,000 = 40GB of data |
| 21 |
N, D = 1000000, 10000 |
| 22 |
USE_FFCV = True |
| 23 |
if not os.path.exists('/tmp/linreg_data.pkl'): |
| 24 |
X = np.random.rand(N, D).astype('float32') |
| 25 |
# Ground-truth vector |
| 26 |
W, b = np.random.rand(D).astype('float32'), np.random.rand() |
| 27 |
# Response variables |
| 28 |
Y = X @ W + b + np.random.randn(N).astype('float32') |
| 29 |
pkl.dump((X, W, b, Y), open('/tmp/linreg_data.pkl', 'wb')) |
| 30 |
elif not USE_FFCV: |
| 31 |
print('Loading from disk...') |
| 32 |
X, W, b, Y = pkl.load(open('/tmp/linreg_data.pkl', 'rb')) |
| 33 |
|
| 34 |
if USE_FFCV and not os.path.exists('/tmp/linreg_data.beton'): |
| 35 |
X, W, b, Y = pkl.load(open('/tmp/linreg_data.pkl', 'rb')) |
| 36 |
class LinearRegressionDataset: |
| 37 |
def __getitem__(self, idx): |
| 38 |
return (X[idx], np.array(Y[idx]).astype('float32')) |
| 39 |
|
| 40 |
def __len__(self): |
| 41 |
return len(X) |
| 42 |
|
| 43 |
writer = DatasetWriter('/tmp/linreg_data.beton', { |
| 44 |
'covariate': NDArrayField(shape=(D,), dtype=np.dtype('float32')), |
| 45 |
'label': NDArrayField(shape=(1,), dtype=np.dtype('float32')), |
| 46 |
}, num_workers=16) |
| 47 |
|
| 48 |
writer.from_indexed_dataset(LinearRegressionDataset()) |
| 49 |
else: |
| 50 |
print('FFCV file already written') |
| 51 |
|
| 52 |
|
| 53 |
### PART 2: actual regression |
| 54 |
|
| 55 |
if not USE_FFCV: |
| 56 |
dataset = TensorDataset(ch.tensor(X), ch.tensor(Y)) |
| 57 |
train_loader = DataLoader(dataset, batch_size=2048, num_workers=8, shuffle=True) |
| 58 |
else: |
| 59 |
train_loader = Loader('/tmp/linreg_data.beton', batch_size=2048, |
| 60 |
num_workers=8, order=OrderOption.QUASI_RANDOM, os_cache=False, |
| 61 |
pipelines={ |
| 62 |
'covariate': [NDArrayDecoder(), ToTensor(), ToDevice(ch.device('cuda:0'))], |
| 63 |
'label': [NDArrayDecoder(), ToTensor(), Squeeze(), ToDevice(ch.device('cuda:0'))] |
| 64 |
}) |
| 65 |
|
| 66 |
# Calculate data mean and variance for normalization |
| 67 |
def calculate_stats(loader, N): |
| 68 |
mean, stdev = 0., 0. |
| 69 |
for x_batch, _ in tqdm(loader): |
| 70 |
mean += x_batch.sum(0) / N |
| 71 |
stdev += x_batch.pow(2).sum(0) / N |
| 72 |
return mean, ch.sqrt(stdev - mean.pow(2)) |
| 73 |
|
| 74 |
mean, stdev = calculate_stats(train_loader, N) |
| 75 |
mean, stdev = mean.cuda(), stdev.cuda() |
| 76 |
w_est, b_est = ch.zeros(D).cuda(), ch.zeros(1).cuda() # Initial guess for W |
| 77 |
num_epochs = 10 # Number of full passes over the data to do |
| 78 |
|
| 79 |
lr = 5e-2 |
| 80 |
for _ in range(num_epochs): |
| 81 |
total_loss, num_examples = 0., 0. |
| 82 |
start_time = time.time() |
| 83 |
for (x_batch, y_batch) in tqdm(train_loader): |
| 84 |
if not USE_FFCV: |
| 85 |
x_batch = x_batch.cuda() |
| 86 |
y_batch = y_batch.cuda() |
| 87 |
# Normalize the data for stability |
| 88 |
x_batch = (x_batch - mean) / stdev |
| 89 |
residual = x_batch @ w_est + b_est - y_batch |
| 90 |
# Gradients |
| 91 |
w_grad = x_batch.T @ residual / x_batch.shape[0] |
| 92 |
b_grad = ch.mean(residual, dim=0) |
| 93 |
w_est = w_est - lr * w_grad |
| 94 |
b_est = b_est - lr * b_grad |
| 95 |
total_loss += residual.pow(2).sum() |
| 96 |
num_examples += x_batch.shape[0] |
| 97 |
print('Epoch time:', time.time() - start_time) |
| 98 |
print(f'Average loss: {total_loss / num_examples:.3f}') |