Debian Codegraph @

examples/cifar/write_datasets.py

raw
1 from argparse import ArgumentParser
2 from typing import List
3 import time
4 import numpy as np
5 from tqdm import tqdm
6
7 import torch as ch
8 import torchvision
9
10 from fastargs import get_current_config
11 from fastargs.decorators import param
12 from fastargs import Param, Section
13 from fastargs.validation import And, OneOf
14
15 from ffcv.writer import DatasetWriter
16 from ffcv.fields import IntField, RGBImageField
17
18 Section('data', 'arguments to give the writer').params(
19 train_dataset=Param(str, 'Where to write the new dataset', required=True),
20 val_dataset=Param(str, 'Where to write the new dataset', required=True),
21 )
22
23 @param('data.train_dataset')
24 @param('data.val_dataset')
25 def main(train_dataset, val_dataset):
26 datasets = {
27 'train': torchvision.datasets.CIFAR10('/tmp', train=True, download=True),
28 'test': torchvision.datasets.CIFAR10('/tmp', train=False, download=True)
29 }
30
31 for (name, ds) in datasets.items():
32 path = train_dataset if name == 'train' else val_dataset
33 writer = DatasetWriter(path, {
34 'image': RGBImageField(),
35 'label': IntField()
36 })
37 writer.from_indexed_dataset(ds)
38
39
40 if __name__ == "__main__":
41 config = get_current_config()
42 parser = ArgumentParser(description='Fast CIFAR-10 training')
43 config.augment_argparse(parser)
44 config.collect_argparse_args(parser)
45 config.validate(mode='stderr')
46 config.summary()
47
48 main()