Debian Codegraph @

examples/docs_examples/custom_transform.py

raw
1 """
2 Example of defining a custom (image) transform using FFCV.
3 For tutorial, see https://docs.ffcv.io/ffcv_examples/custom_transforms.html.
4
5 """
6 import time
7 import numpy as np
8 import torchvision
9
10 from ffcv.fields import IntField, RGBImageField
11 from ffcv.fields.decoders import SimpleRGBImageDecoder
12 from ffcv.loader import Loader, OrderOption
13 from ffcv.pipeline.compiler import Compiler
14 from ffcv.pipeline.operation import Operation, AllocationQuery
15 from ffcv.transforms import ToTensor
16 from ffcv.writer import DatasetWriter
17 from dataclasses import replace
18
19 class PickACorner(Operation):
20 def generate_code(self):
21 parallel_range = Compiler.get_iterator()
22 def pick_a_corner(images, dst):
23 which_corner = np.random.rand(images.shape[0])
24 for i in parallel_range(images.shape[0]):
25 if which_corner[i] == 0:
26 dst[i] = images[i,:images.shape[1]//2, :images.shape[2]//2]
27 else:
28 dst[i] = images[i,-images.shape[1]//2:, -images.shape[2]//2:]
29
30 return dst
31
32 pick_a_corner.is_parallel = True
33 return pick_a_corner
34
35 def declare_state_and_memory(self, previous_state):
36 h, w, c = previous_state.shape
37 new_shape = (h // 2, w // 2, c)
38
39 new_state = replace(previous_state, shape=new_shape)
40 mem_allocation = AllocationQuery(new_shape, previous_state.dtype)
41 return (new_state, mem_allocation)
42
43 # Step 1: Create an FFCV-compatible CIFAR-10 dataset
44 ds = torchvision.datasets.CIFAR10('/tmp', train=True, download=True)
45 writer = DatasetWriter('/tmp/cifar.beton', {
46 'image': RGBImageField(),
47 'label': IntField()
48 })
49 writer.from_indexed_dataset(ds)
50
51 # Step 2: Create data loaders
52 BATCH_SIZE = 512
53 # Create loaders
54 image_pipelines = {
55 'with': [SimpleRGBImageDecoder(), PickACorner(), ToTensor()],
56 'without': [SimpleRGBImageDecoder(), ToTensor()]
57 }
58
59 for name, pipeline in image_pipelines.items():
60 loader = Loader(f'/tmp/cifar.beton', batch_size=BATCH_SIZE,
61 num_workers=16, order=OrderOption.RANDOM,
62 drop_last=True, pipelines={'image': pipeline})
63
64 # First epoch includes compilation time
65 for ims, labs in loader: pass
66 start_time = time.time()
67 for _ in range(100):
68 for ims, labs in loader: pass
69 print(f'Method: {name} | Shape: {ims.shape} | Time per epoch: {(time.time() - start_time) / 100:.5f}s')