from argparse import ArgumentParser from typing import List import time import numpy as np from tqdm import tqdm import torch as ch import torchvision from fastargs import get_current_config from fastargs.decorators import param from fastargs import Param, Section from fastargs.validation import And, OneOf from ffcv.writer import DatasetWriter from ffcv.fields import IntField, RGBImageField Section('data', 'arguments to give the writer').params( train_dataset=Param(str, 'Where to write the new dataset', required=True), val_dataset=Param(str, 'Where to write the new dataset', required=True), ) @param('data.train_dataset') @param('data.val_dataset') def main(train_dataset, val_dataset): datasets = { 'train': torchvision.datasets.CIFAR10('/tmp', train=True, download=True), 'test': torchvision.datasets.CIFAR10('/tmp', train=False, download=True) } for (name, ds) in datasets.items(): path = train_dataset if name == 'train' else val_dataset writer = DatasetWriter(path, { 'image': RGBImageField(), 'label': IntField() }) writer.from_indexed_dataset(ds) if __name__ == "__main__": config = get_current_config() parser = ArgumentParser(description='Fast CIFAR-10 training') config.augment_argparse(parser) config.collect_argparse_args(parser) config.validate(mode='stderr') config.summary() main()