Debian Codegraph @

examples/cifar/train_cifar.py

raw
1 """
2 Fast training script for CIFAR-10 using FFCV.
3 For tutorial, see https://docs.ffcv.io/ffcv_examples/cifar10.html.
4
5 First, from the same directory, run:
6
7 `python write_datasets.py --data.train_dataset [TRAIN_PATH] \
8 --data.val_dataset [VAL_PATH]`
9
10 to generate the FFCV-formatted versions of CIFAR.
11
12 Then, simply run this to train models with default hyperparameters:
13
14 `python train_cifar.py --config-file default_config.yaml`
15
16 You can override arguments as follows:
17
18 `python train_cifar.py --config-file default_config.yaml \
19 --training.lr 0.2 --training.num_workers 4 ... [etc]`
20
21 or by using a different config file.
22 """
23 from argparse import ArgumentParser
24 from typing import List
25 import time
26 import numpy as np
27 from tqdm import tqdm
28
29 import torch as ch
30 from torch.cuda.amp import GradScaler, autocast
31 from torch.nn import CrossEntropyLoss, Conv2d, BatchNorm2d
32 from torch.optim import SGD, lr_scheduler
33 import torchvision
34
35 from fastargs import get_current_config, Param, Section
36 from fastargs.decorators import param
37 from fastargs.validation import And, OneOf
38
39 from ffcv.fields import IntField, RGBImageField
40 from ffcv.fields.decoders import IntDecoder, SimpleRGBImageDecoder
41 from ffcv.loader import Loader, OrderOption
42 from ffcv.pipeline.operation import Operation
43 from ffcv.transforms import RandomHorizontalFlip, Cutout, \
44 RandomTranslate, Convert, ToDevice, ToTensor, ToTorchImage
45 from ffcv.transforms.common import Squeeze
46 from ffcv.writer import DatasetWriter
47
48 Section('training', 'Hyperparameters').params(
49 lr=Param(float, 'The learning rate to use', required=True),
50 epochs=Param(int, 'Number of epochs to run for', required=True),
51 lr_peak_epoch=Param(int, 'Peak epoch for cyclic lr', required=True),
52 batch_size=Param(int, 'Batch size', default=512),
53 momentum=Param(float, 'Momentum for SGD', default=0.9),
54 weight_decay=Param(float, 'l2 weight decay', default=5e-4),
55 label_smoothing=Param(float, 'Value of label smoothing', default=0.1),
56 num_workers=Param(int, 'The number of workers', default=8),
57 lr_tta=Param(bool, 'Test time augmentation by averaging with horizontally flipped version', default=True)
58 )
59
60 Section('data', 'data related stuff').params(
61 train_dataset=Param(str, '.dat file to use for training', required=True),
62 val_dataset=Param(str, '.dat file to use for validation', required=True),
63 )
64
65 @param('data.train_dataset')
66 @param('data.val_dataset')
67 @param('training.batch_size')
68 @param('training.num_workers')
69 def make_dataloaders(train_dataset=None, val_dataset=None, batch_size=None, num_workers=None):
70 paths = {
71 'train': train_dataset,
72 'test': val_dataset
73
74 }
75
76 start_time = time.time()
77 CIFAR_MEAN = [125.307, 122.961, 113.8575]
78 CIFAR_STD = [51.5865, 50.847, 51.255]
79 loaders = {}
80
81 for name in ['train', 'test']:
82 label_pipeline: List[Operation] = [IntDecoder(), ToTensor(), ToDevice(ch.device('cuda:0')), Squeeze()]
83 image_pipeline: List[Operation] = [SimpleRGBImageDecoder()]
84 if name == 'train':
85 image_pipeline.extend([
86 RandomHorizontalFlip(),
87 RandomTranslate(padding=2, fill=tuple(map(int, CIFAR_MEAN))),
88 Cutout(4, tuple(map(int, CIFAR_MEAN))),
89 ])
90 image_pipeline.extend([
91 ToTensor(),
92 ToDevice(ch.device('cuda:0'), non_blocking=True),
93 ToTorchImage(),
94 Convert(ch.float16),
95 torchvision.transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
96 ])
97
98 ordering = OrderOption.RANDOM if name == 'train' else OrderOption.SEQUENTIAL
99
100 loaders[name] = Loader(paths[name], batch_size=batch_size, num_workers=num_workers,
101 order=ordering, drop_last=(name == 'train'),
102 pipelines={'image': image_pipeline, 'label': label_pipeline})
103
104 return loaders, start_time
105
106 # Model (from KakaoBrain: https://github.com/wbaek/torchskeleton)
107 class Mul(ch.nn.Module):
108 def __init__(self, weight):
109 super(Mul, self).__init__()
110 self.weight = weight
111 def forward(self, x): return x * self.weight
112
113 class Flatten(ch.nn.Module):
114 def forward(self, x): return x.view(x.size(0), -1)
115
116 class Residual(ch.nn.Module):
117 def __init__(self, module):
118 super(Residual, self).__init__()
119 self.module = module
120 def forward(self, x): return x + self.module(x)
121
122 def conv_bn(channels_in, channels_out, kernel_size=3, stride=1, padding=1, groups=1):
123 return ch.nn.Sequential(
124 ch.nn.Conv2d(channels_in, channels_out, kernel_size=kernel_size,
125 stride=stride, padding=padding, groups=groups, bias=False),
126 ch.nn.BatchNorm2d(channels_out),
127 ch.nn.ReLU(inplace=True)
128 )
129
130 def construct_model():
131 num_class = 10
132 model = ch.nn.Sequential(
133 conv_bn(3, 64, kernel_size=3, stride=1, padding=1),
134 conv_bn(64, 128, kernel_size=5, stride=2, padding=2),
135 Residual(ch.nn.Sequential(conv_bn(128, 128), conv_bn(128, 128))),
136 conv_bn(128, 256, kernel_size=3, stride=1, padding=1),
137 ch.nn.MaxPool2d(2),
138 Residual(ch.nn.Sequential(conv_bn(256, 256), conv_bn(256, 256))),
139 conv_bn(256, 128, kernel_size=3, stride=1, padding=0),
140 ch.nn.AdaptiveMaxPool2d((1, 1)),
141 Flatten(),
142 ch.nn.Linear(128, num_class, bias=False),
143 Mul(0.2)
144 )
145 model = model.to(memory_format=ch.channels_last).cuda()
146 return model
147
148 @param('training.lr')
149 @param('training.epochs')
150 @param('training.momentum')
151 @param('training.weight_decay')
152 @param('training.label_smoothing')
153 @param('training.lr_peak_epoch')
154 def train(model, loaders, lr=None, epochs=None, label_smoothing=None,
155 momentum=None, weight_decay=None, lr_peak_epoch=None):
156 opt = SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)
157 iters_per_epoch = len(loaders['train'])
158 # Cyclic LR with single triangle
159 lr_schedule = np.interp(np.arange((epochs+1) * iters_per_epoch),
160 [0, lr_peak_epoch * iters_per_epoch, epochs * iters_per_epoch],
161 [0, 1, 0])
162 scheduler = lr_scheduler.LambdaLR(opt, lr_schedule.__getitem__)
163 scaler = GradScaler()
164 loss_fn = CrossEntropyLoss(label_smoothing=label_smoothing)
165
166 for _ in range(epochs):
167 for ims, labs in tqdm(loaders['train']):
168 opt.zero_grad(set_to_none=True)
169 with autocast():
170 out = model(ims)
171 loss = loss_fn(out, labs)
172
173 scaler.scale(loss).backward()
174 scaler.step(opt)
175 scaler.update()
176 scheduler.step()
177
178 @param('training.lr_tta')
179 def evaluate(model, loaders, lr_tta=False):
180 model.eval()
181 with ch.no_grad():
182 for name in ['train', 'test']:
183 total_correct, total_num = 0., 0.
184 for ims, labs in tqdm(loaders[name]):
185 with autocast():
186 out = model(ims)
187 if lr_tta:
188 out += model(ims.flip(-1))
189 total_correct += out.argmax(1).eq(labs).sum().cpu().item()
190 total_num += ims.shape[0]
191 print(f'{name} accuracy: {total_correct / total_num * 100:.1f}%')
192
193 if __name__ == "__main__":
194 config = get_current_config()
195 parser = ArgumentParser(description='Fast CIFAR-10 training')
196 config.augment_argparse(parser)
197 # Also loads from args.config_path if provided
198 config.collect_argparse_args(parser)
199 config.validate(mode='stderr')
200 config.summary()
201
202 loaders, start_time = make_dataloaders()
203 model = construct_model()
204 train(model, loaders)
205 print(f'Total time: {time.time() - start_time:.5f}')
206 evaluate(model, loaders)