| 1 |
from argparse import ArgumentParser |
| 2 |
from typing import List |
| 3 |
import time |
| 4 |
import numpy as np |
| 5 |
from tqdm import tqdm |
| 6 |
|
| 7 |
import torch as ch |
| 8 |
import torchvision |
| 9 |
|
| 10 |
from fastargs import get_current_config |
| 11 |
from fastargs.decorators import param |
| 12 |
from fastargs import Param, Section |
| 13 |
from fastargs.validation import And, OneOf |
| 14 |
|
| 15 |
from ffcv.writer import DatasetWriter |
| 16 |
from ffcv.fields import IntField, RGBImageField |
| 17 |
|
| 18 |
Section('data', 'arguments to give the writer').params( |
| 19 |
train_dataset=Param(str, 'Where to write the new dataset', required=True), |
| 20 |
val_dataset=Param(str, 'Where to write the new dataset', required=True), |
| 21 |
) |
| 22 |
|
| 23 |
@param('data.train_dataset') |
| 24 |
@param('data.val_dataset') |
| 25 |
def main(train_dataset, val_dataset): |
| 26 |
datasets = { |
| 27 |
'train': torchvision.datasets.CIFAR10('/tmp', train=True, download=True), |
| 28 |
'test': torchvision.datasets.CIFAR10('/tmp', train=False, download=True) |
| 29 |
} |
| 30 |
|
| 31 |
for (name, ds) in datasets.items(): |
| 32 |
path = train_dataset if name == 'train' else val_dataset |
| 33 |
writer = DatasetWriter(path, { |
| 34 |
'image': RGBImageField(), |
| 35 |
'label': IntField() |
| 36 |
}) |
| 37 |
writer.from_indexed_dataset(ds) |
| 38 |
|
| 39 |
|
| 40 |
if __name__ == "__main__": |
| 41 |
config = get_current_config() |
| 42 |
parser = ArgumentParser(description='Fast CIFAR-10 training') |
| 43 |
config.augment_argparse(parser) |
| 44 |
config.collect_argparse_args(parser) |
| 45 |
config.validate(mode='stderr') |
| 46 |
config.summary() |
| 47 |
|
| 48 |
main() |