Debian Codegraph @

.cargo/vendor/exr/src/block/writer.rs

raw
1 //! Composable structures to handle writing an image.
2
3
4 use std::fmt::Debug;
5 use std::io::Seek;
6 use std::iter::Peekable;
7 use std::ops::Not;
8
9 use smallvec::alloc::collections::BTreeMap;
10 use crate::block::UncompressedBlock;
11 use crate::block::chunk::Chunk;
12 use crate::error::{Error, Result, UnitResult, usize_to_u64};
13 use crate::io::{Data, Tracking, Write};
14 use crate::meta::{Headers, MetaData, OffsetTables};
15 use crate::meta::attribute::LineOrder;
16
17 /// Write an exr file by writing one chunk after another in a closure.
18 /// In the closure, you are provided a chunk writer, which should be used to write all the chunks.
19 /// Assumes the your write destination is buffered.
20 pub fn write_chunks_with<W: Write + Seek>(
21 buffered_write: W, headers: Headers, pedantic: bool,
22 write_chunks: impl FnOnce(MetaData, &mut ChunkWriter<W>) -> UnitResult
23 ) -> UnitResult {
24 // this closure approach ensures that after writing all chunks, the file is always completed and checked and flushed
25 let (meta, mut writer) = ChunkWriter::new_for_buffered(buffered_write, headers, pedantic)?;
26 write_chunks(meta, &mut writer)?;
27 writer.complete_meta_data()
28 }
29
30 /// Can consume compressed pixel chunks, writing them a file.
31 /// Use `sequential_blocks_compressor` or `parallel_blocks_compressor` to compress your data,
32 /// or use `compress_all_blocks_sequential` or `compress_all_blocks_parallel`.
33 /// Use `on_progress` to obtain a new writer
34 /// that triggers a callback for each block.
35 // #[must_use]
36 #[derive(Debug)]
37 #[must_use]
38 pub struct ChunkWriter<W> {
39 header_count: usize,
40 byte_writer: Tracking<W>,
41 chunk_indices_byte_location: std::ops::Range<usize>,
42 chunk_indices_increasing_y: OffsetTables,
43 chunk_count: usize, // TODO compose?
44 }
45
46 /// A new writer that triggers a callback
47 /// for each block written to the inner writer.
48 #[derive(Debug)]
49 #[must_use]
50 pub struct OnProgressChunkWriter<'w, W, F> {
51 chunk_writer: &'w mut W,
52 written_chunks: usize,
53 on_progress: F,
54 }
55
56 /// Write chunks to a byte destination.
57 /// Then write each chunk with `writer.write_chunk(chunk)`.
58 pub trait ChunksWriter: Sized {
59
60 /// The total number of chunks that the complete file will contain.
61 fn total_chunks_count(&self) -> usize;
62
63 /// Any more calls will result in an error and have no effect.
64 /// If writing results in an error, the file and the writer
65 /// may remain in an invalid state and should not be used further.
66 /// Errors when the chunk at this index was already written.
67 fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult;
68
69 /// Obtain a new writer that calls the specified closure for each block that is written to this writer.
70 fn on_progress<F>(&mut self, on_progress: F) -> OnProgressChunkWriter<'_, Self, F> where F: FnMut(f64) {
71 OnProgressChunkWriter { chunk_writer: self, written_chunks: 0, on_progress }
72 }
73
74 /// Obtain a new writer that can compress blocks to chunks, which are then passed to this writer.
75 fn sequential_blocks_compressor<'w>(&'w mut self, meta: &'w MetaData) -> SequentialBlocksCompressor<'w, Self> {
76 SequentialBlocksCompressor::new(meta, self)
77 }
78
79 #[cfg(feature = "rayon")]
80 /// Obtain a new writer that can compress blocks to chunks on multiple threads, which are then passed to this writer.
81 /// Returns none if the sequential compressor should be used instead (thread pool creation failure or too large performance overhead).
82 fn parallel_blocks_compressor<'w>(&'w mut self, meta: &'w MetaData) -> Option<ParallelBlocksCompressor<'w, Self>> {
83 ParallelBlocksCompressor::new(meta, self)
84 }
85
86 /// Compresses all blocks to the file.
87 /// The index of the block must be in increasing line order within the header.
88 /// Obtain iterator with `MetaData::collect_ordered_blocks(...)` or similar methods.
89 fn compress_all_blocks_sequential(mut self, meta: &MetaData, blocks: impl Iterator<Item=(usize, UncompressedBlock)>) -> UnitResult {
90 let mut writer = self.sequential_blocks_compressor(meta);
91
92 // TODO check block order if line order is not unspecified!
93 for (index_in_header_increasing_y, block) in blocks {
94 writer.compress_block(index_in_header_increasing_y, block)?;
95 }
96
97 // TODO debug_assert_eq!(self.is_complete());
98 Ok(())
99 }
100
101 #[cfg(feature = "rayon")]
102 /// Compresses all blocks to the file.
103 /// The index of the block must be in increasing line order within the header.
104 /// Obtain iterator with `MetaData::collect_ordered_blocks(...)` or similar methods.
105 /// Will fallback to sequential processing where threads are not available, or where it would not speed up the process.
106 fn compress_all_blocks_parallel(mut self, meta: &MetaData, blocks: impl Iterator<Item=(usize, UncompressedBlock)>) -> UnitResult {
107 let mut parallel_writer = match self.parallel_blocks_compressor(meta) {
108 None => return self.compress_all_blocks_sequential(meta, blocks),
109 Some(writer) => writer,
110 };
111
112 // TODO check block order if line order is not unspecified!
113 for (index_in_header_increasing_y, block) in blocks {
114 parallel_writer.add_block_to_compression_queue(index_in_header_increasing_y, block)?;
115 }
116
117 // TODO debug_assert_eq!(self.is_complete());
118 Ok(())
119 }
120 }
121
122
123 impl<W> ChunksWriter for ChunkWriter<W> where W: Write + Seek {
124
125 /// The total number of chunks that the complete file will contain.
126 fn total_chunks_count(&self) -> usize { self.chunk_count }
127
128 /// Any more calls will result in an error and have no effect.
129 /// If writing results in an error, the file and the writer
130 /// may remain in an invalid state and should not be used further.
131 /// Errors when the chunk at this index was already written.
132 fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult {
133 let header_chunk_indices = &mut self.chunk_indices_increasing_y[chunk.layer_index];
134
135 if index_in_header_increasing_y >= header_chunk_indices.len() {
136 return Err(Error::invalid("too large chunk index"));
137 }
138
139 let chunk_index_slot = &mut header_chunk_indices[index_in_header_increasing_y];
140 if *chunk_index_slot != 0 {
141 return Err(Error::invalid(format!("chunk at index {} is already written", index_in_header_increasing_y)));
142 }
143
144 *chunk_index_slot = usize_to_u64(self.byte_writer.byte_position(), "seek position")?;
145 chunk.write(&mut self.byte_writer, self.header_count)?;
146 Ok(())
147 }
148 }
149
150 impl<W> ChunkWriter<W> where W: Write + Seek {
151 // -- the following functions are private, because they must be called in a strict order --
152
153 /// Writes the meta data and zeroed offset tables as a placeholder.
154 fn new_for_buffered(buffered_byte_writer: W, headers: Headers, pedantic: bool) -> Result<(MetaData, Self)> {
155 let mut write = Tracking::new(buffered_byte_writer);
156 let requirements = MetaData::write_validating_to_buffered(&mut write, headers.as_slice(), pedantic)?;
157
158 // TODO: use increasing line order where possible, but this requires us to know whether we want to be parallel right now
159 /*// if non-parallel compression, we always use increasing order anyways
160 if !parallel || !has_compression {
161 for header in &mut headers {
162 if header.line_order == LineOrder::Unspecified {
163 header.line_order = LineOrder::Increasing;
164 }
165 }
166 }*/
167
168 let offset_table_size: usize = headers.iter().map(|header| header.chunk_count).sum();
169
170 let offset_table_start_byte = write.byte_position();
171 let offset_table_end_byte = write.byte_position() + offset_table_size * u64::BYTE_SIZE;
172
173 // skip offset tables, filling with 0, will be updated after the last chunk has been written
174 write.seek_write_to(offset_table_end_byte)?;
175
176 let header_count = headers.len();
177 let chunk_indices_increasing_y = headers.iter()
178 .map(|header| vec![0_u64; header.chunk_count]).collect();
179
180 let meta_data = MetaData { requirements, headers };
181
182 Ok((meta_data, ChunkWriter {
183 header_count,
184 byte_writer: write,
185 chunk_count: offset_table_size,
186 chunk_indices_byte_location: offset_table_start_byte .. offset_table_end_byte,
187 chunk_indices_increasing_y,
188 }))
189 }
190
191 /// Seek back to the meta data, write offset tables, and flush the byte writer.
192 /// Leaves the writer seeked to the middle of the file, therefore we drop it.
193 fn complete_meta_data(mut self) -> UnitResult {
194 if self.chunk_indices_increasing_y.iter().flatten().any(|&index| index == 0) {
195 return Err(Error::invalid("some chunks are not written yet"))
196 }
197
198 // write all offset tables
199 debug_assert_ne!(self.byte_writer.byte_position(), self.chunk_indices_byte_location.end, "offset table has already been updated");
200 self.byte_writer.seek_write_to(self.chunk_indices_byte_location.start)?;
201
202 for table in self.chunk_indices_increasing_y {
203 u64::write_slice_le(&mut self.byte_writer, table.as_slice())?;
204 }
205
206 self.byte_writer.flush()?; // make sure we catch all (possibly delayed) io errors before returning
207 Ok(())
208 }
209
210 }
211
212
213 impl<'w, W, F> ChunksWriter for OnProgressChunkWriter<'w, W, F> where W: 'w + ChunksWriter, F: FnMut(f64) {
214 fn total_chunks_count(&self) -> usize {
215 self.chunk_writer.total_chunks_count()
216 }
217
218 fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult {
219 let total_chunks = self.total_chunks_count();
220 let on_progress = &mut self.on_progress;
221
222 // guarantee on_progress being called with 0 once
223 if self.written_chunks == 0 { on_progress(0.0); }
224
225 self.chunk_writer.write_chunk(index_in_header_increasing_y, chunk)?;
226
227 self.written_chunks += 1;
228
229 on_progress({
230 // guarantee finishing with progress 1.0 for last block at least once, float division might slightly differ from 1.0
231 if self.written_chunks == total_chunks { 1.0 }
232 else { self.written_chunks as f64 / total_chunks as f64 }
233 });
234
235 Ok(())
236 }
237 }
238
239
240 /// Write blocks that appear in any order and reorder them before writing.
241 #[derive(Debug)]
242 #[must_use]
243 pub struct SortedBlocksWriter<'w, W> {
244 chunk_writer: &'w mut W,
245 pending_chunks: BTreeMap<usize, (usize, Chunk)>,
246 unwritten_chunk_indices: Peekable<std::ops::Range<usize>>,
247 requires_sorting: bool, // using this instead of Option, because of borrowing
248 }
249
250
251 impl<'w, W> SortedBlocksWriter<'w, W> where W: ChunksWriter {
252
253 /// New sorting writer. Returns `None` if sorting is not required.
254 pub fn new(meta_data: &MetaData, chunk_writer: &'w mut W) -> SortedBlocksWriter<'w, W> {
255 let requires_sorting = meta_data.headers.iter()
256 .any(|header| header.line_order != LineOrder::Unspecified);
257
258 let total_chunk_count = chunk_writer.total_chunks_count();
259
260 SortedBlocksWriter {
261 pending_chunks: BTreeMap::new(),
262 unwritten_chunk_indices: (0 .. total_chunk_count).peekable(),
263 requires_sorting,
264 chunk_writer
265 }
266 }
267
268 /// Write the chunk or stash it. In the closure, write all chunks that can be written now.
269 pub fn write_or_stash_chunk(&mut self, chunk_index_in_file: usize, chunk_y_index: usize, chunk: Chunk) -> UnitResult {
270 if self.requires_sorting.not() {
271 return self.chunk_writer.write_chunk(chunk_y_index, chunk);
272 }
273
274 // write this chunk now if possible
275 if self.unwritten_chunk_indices.peek() == Some(&chunk_index_in_file){
276 self.chunk_writer.write_chunk(chunk_y_index, chunk)?;
277 self.unwritten_chunk_indices.next().expect("peeked chunk index is missing");
278
279 // write all pending blocks that are immediate successors of this block
280 while let Some((next_chunk_y_index, next_chunk)) = self
281 .unwritten_chunk_indices.peek().cloned()
282 .and_then(|id| self.pending_chunks.remove(&id))
283 {
284 self.chunk_writer.write_chunk(next_chunk_y_index, next_chunk)?;
285 self.unwritten_chunk_indices.next().expect("peeked chunk index is missing");
286 }
287 }
288
289 else {
290 // the argument block is not to be written now,
291 // and all the pending blocks are not next up either,
292 // so just stash this block
293 self.pending_chunks.insert(chunk_index_in_file, (chunk_y_index, chunk));
294 }
295
296 Ok(())
297 }
298
299 /// Where the chunks will be written to.
300 pub fn inner_chunks_writer(&self) -> &W {
301 &self.chunk_writer
302 }
303 }
304
305
306
307 /// Compress blocks to a chunk writer in this thread.
308 #[derive(Debug)]
309 #[must_use]
310 pub struct SequentialBlocksCompressor<'w, W> {
311 meta: &'w MetaData,
312 chunks_writer: &'w mut W,
313 }
314
315 impl<'w, W> SequentialBlocksCompressor<'w, W> where W: 'w + ChunksWriter {
316
317 /// New blocks writer.
318 pub fn new(meta: &'w MetaData, chunks_writer: &'w mut W) -> Self { Self { meta, chunks_writer, } }
319
320 /// This is where the compressed blocks are written to.
321 pub fn inner_chunks_writer(&'w self) -> &'w W { self.chunks_writer }
322
323 /// Compress a single block immediately. The index of the block must be in increasing line order.
324 pub fn compress_block(&mut self, index_in_header_increasing_y: usize, block: UncompressedBlock) -> UnitResult {
325 self.chunks_writer.write_chunk(
326 index_in_header_increasing_y,
327 block.compress_to_chunk(&self.meta.headers)?
328 )
329 }
330 }
331
332
333
334 /// Compress blocks to a chunk writer with multiple threads.
335 #[cfg(feature = "rayon")]
336 #[derive(Debug)]
337 #[must_use]
338 pub struct ParallelBlocksCompressor<'w, W> {
339 meta: &'w MetaData,
340 sorted_writer: SortedBlocksWriter<'w, W>,
341
342 sender: std::sync::mpsc::Sender<Result<(usize, usize, Chunk)>>,
343 receiver: std::sync::mpsc::Receiver<Result<(usize, usize, Chunk)>>,
344 pool: rayon_core::ThreadPool,
345
346 currently_compressing_count: usize,
347 written_chunk_count: usize, // used to check for last chunk
348 max_threads: usize,
349 next_incoming_chunk_index: usize, // used to remember original chunk order
350 }
351
352 #[cfg(feature = "rayon")]
353 impl<'w, W> ParallelBlocksCompressor<'w, W> where W: 'w + ChunksWriter {
354
355 /// New blocks writer. Returns none if sequential compression should be used.
356 /// Use `new_with_thread_pool` to customize the threadpool.
357 pub fn new(meta: &'w MetaData, chunks_writer: &'w mut W) -> Option<Self> {
358 Self::new_with_thread_pool(meta, chunks_writer, ||{
359 rayon_core::ThreadPoolBuilder::new()
360 .thread_name(|index| format!("OpenEXR Block Compressor Thread #{}", index))
361 .build()
362 })
363 }
364
365 /// New blocks writer. Returns none if sequential compression should be used.
366 pub fn new_with_thread_pool<CreatePool>(
367 meta: &'w MetaData, chunks_writer: &'w mut W, try_create_thread_pool: CreatePool)
368 -> Option<Self>
369 where CreatePool: FnOnce() -> std::result::Result<rayon_core::ThreadPool, rayon_core::ThreadPoolBuildError>
370 {
371 use crate::compression::Compression;
372
373 let is_entirely_uncompressed = meta.headers.iter()
374 .all(|head|head.compression == Compression::Uncompressed);
375
376 if is_entirely_uncompressed {
377 return None;
378 }
379
380 // in case thread pool creation fails (for example on WASM currently),
381 // we revert to sequential compression
382 let pool = match try_create_thread_pool() {
383 Ok(pool) => pool,
384
385 // TODO print warning?
386 Err(_) => return None,
387 };
388
389 let max_threads = pool.current_num_threads().max(1).min(chunks_writer.total_chunks_count()) + 2; // ca one block for each thread at all times
390 let (send, recv) = std::sync::mpsc::channel(); // TODO bounded channel simplifies logic?
391
392 Some(Self {
393 sorted_writer: SortedBlocksWriter::new(meta, chunks_writer),
394 next_incoming_chunk_index: 0,
395 currently_compressing_count: 0,
396 written_chunk_count: 0,
397 sender: send,
398 receiver: recv,
399 max_threads,
400 pool,
401 meta,
402 })
403 }
404
405 /// This is where the compressed blocks are written to.
406 pub fn inner_chunks_writer(&'w self) -> &'w W { self.sorted_writer.inner_chunks_writer() }
407
408 // private, as may underflow counter in release mode
409 fn write_next_queued_chunk(&mut self) -> UnitResult {
410 debug_assert!(self.currently_compressing_count > 0, "cannot wait for chunks as there are none left");
411
412 let some_compressed_chunk = self.receiver.recv()
413 .expect("cannot receive compressed block");
414
415 self.currently_compressing_count -= 1;
416 let (chunk_file_index, chunk_y_index, chunk) = some_compressed_chunk?;
417 self.sorted_writer.write_or_stash_chunk(chunk_file_index, chunk_y_index, chunk)?;
418
419 self.written_chunk_count += 1;
420 Ok(())
421 }
422
423 /// Wait until all currently compressing chunks in the compressor have been written.
424 pub fn write_all_queued_chunks(&mut self) -> UnitResult {
425 while self.currently_compressing_count > 0 {
426 self.write_next_queued_chunk()?;
427 }
428
429 debug_assert_eq!(self.currently_compressing_count, 0, "counter does not match block count");
430 Ok(())
431 }
432
433 /// Add a single block to the compressor queue. The index of the block must be in increasing line order.
434 /// When calling this function for the last block, this method waits until all the blocks have been written.
435 /// This only works when you write as many blocks as the image expects, otherwise you can use `wait_for_all_remaining_chunks`.
436 /// Waits for a block from the queue to be written, if the queue already has enough items.
437 pub fn add_block_to_compression_queue(&mut self, index_in_header_increasing_y: usize, block: UncompressedBlock) -> UnitResult {
438
439 // if pipe is full, block to wait for a slot to free up
440 if self.currently_compressing_count >= self.max_threads {
441 self.write_next_queued_chunk()?;
442 }
443
444 // add the argument chunk to the compression queueue
445 let index_in_file = self.next_incoming_chunk_index;
446 let sender = self.sender.clone();
447 let meta = self.meta.clone();
448
449 self.pool.spawn(move ||{
450 let compressed_or_err = block.compress_to_chunk(&meta.headers);
451
452 // by now, decompressing could have failed in another thread.
453 // the error is then already handled, so we simply
454 // don't send the decompressed block and do nothing
455 let _ = sender.send(compressed_or_err.map(move |compressed| (index_in_file, index_in_header_increasing_y, compressed)));
456 });
457
458 self.currently_compressing_count += 1;
459 self.next_incoming_chunk_index += 1;
460
461 // if this is the last chunk, wait for all chunks to complete before returning
462 if self.written_chunk_count + self.currently_compressing_count == self.inner_chunks_writer().total_chunks_count() {
463 self.write_all_queued_chunks()?;
464 debug_assert_eq!(
465 self.written_chunk_count, self.inner_chunks_writer().total_chunks_count(),
466 "written chunk count mismatch"
467 );
468 }
469
470
471 Ok(())
472 }
473 }
474
475
476