| 1 |
//! Composable structures to handle writing an image. |
| 2 |
|
| 3 |
|
| 4 |
use std::fmt::Debug; |
| 5 |
use std::io::Seek; |
| 6 |
use std::iter::Peekable; |
| 7 |
use std::ops::Not; |
| 8 |
|
| 9 |
use smallvec::alloc::collections::BTreeMap; |
| 10 |
use crate::block::UncompressedBlock; |
| 11 |
use crate::block::chunk::Chunk; |
| 12 |
use crate::error::{Error, Result, UnitResult, usize_to_u64}; |
| 13 |
use crate::io::{Data, Tracking, Write}; |
| 14 |
use crate::meta::{Headers, MetaData, OffsetTables}; |
| 15 |
use crate::meta::attribute::LineOrder; |
| 16 |
|
| 17 |
/// Write an exr file by writing one chunk after another in a closure. |
| 18 |
/// In the closure, you are provided a chunk writer, which should be used to write all the chunks. |
| 19 |
/// Assumes the your write destination is buffered. |
| 20 |
pub fn write_chunks_with<W: Write + Seek>( |
| 21 |
buffered_write: W, headers: Headers, pedantic: bool, |
| 22 |
write_chunks: impl FnOnce(MetaData, &mut ChunkWriter<W>) -> UnitResult |
| 23 |
) -> UnitResult { |
| 24 |
// this closure approach ensures that after writing all chunks, the file is always completed and checked and flushed |
| 25 |
let (meta, mut writer) = ChunkWriter::new_for_buffered(buffered_write, headers, pedantic)?; |
| 26 |
write_chunks(meta, &mut writer)?; |
| 27 |
writer.complete_meta_data() |
| 28 |
} |
| 29 |
|
| 30 |
/// Can consume compressed pixel chunks, writing them a file. |
| 31 |
/// Use `sequential_blocks_compressor` or `parallel_blocks_compressor` to compress your data, |
| 32 |
/// or use `compress_all_blocks_sequential` or `compress_all_blocks_parallel`. |
| 33 |
/// Use `on_progress` to obtain a new writer |
| 34 |
/// that triggers a callback for each block. |
| 35 |
// #[must_use] |
| 36 |
#[derive(Debug)] |
| 37 |
#[must_use] |
| 38 |
pub struct ChunkWriter<W> { |
| 39 |
header_count: usize, |
| 40 |
byte_writer: Tracking<W>, |
| 41 |
chunk_indices_byte_location: std::ops::Range<usize>, |
| 42 |
chunk_indices_increasing_y: OffsetTables, |
| 43 |
chunk_count: usize, // TODO compose? |
| 44 |
} |
| 45 |
|
| 46 |
/// A new writer that triggers a callback |
| 47 |
/// for each block written to the inner writer. |
| 48 |
#[derive(Debug)] |
| 49 |
#[must_use] |
| 50 |
pub struct OnProgressChunkWriter<'w, W, F> { |
| 51 |
chunk_writer: &'w mut W, |
| 52 |
written_chunks: usize, |
| 53 |
on_progress: F, |
| 54 |
} |
| 55 |
|
| 56 |
/// Write chunks to a byte destination. |
| 57 |
/// Then write each chunk with `writer.write_chunk(chunk)`. |
| 58 |
pub trait ChunksWriter: Sized { |
| 59 |
|
| 60 |
/// The total number of chunks that the complete file will contain. |
| 61 |
fn total_chunks_count(&self) -> usize; |
| 62 |
|
| 63 |
/// Any more calls will result in an error and have no effect. |
| 64 |
/// If writing results in an error, the file and the writer |
| 65 |
/// may remain in an invalid state and should not be used further. |
| 66 |
/// Errors when the chunk at this index was already written. |
| 67 |
fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult; |
| 68 |
|
| 69 |
/// Obtain a new writer that calls the specified closure for each block that is written to this writer. |
| 70 |
fn on_progress<F>(&mut self, on_progress: F) -> OnProgressChunkWriter<'_, Self, F> where F: FnMut(f64) { |
| 71 |
OnProgressChunkWriter { chunk_writer: self, written_chunks: 0, on_progress } |
| 72 |
} |
| 73 |
|
| 74 |
/// Obtain a new writer that can compress blocks to chunks, which are then passed to this writer. |
| 75 |
fn sequential_blocks_compressor<'w>(&'w mut self, meta: &'w MetaData) -> SequentialBlocksCompressor<'w, Self> { |
| 76 |
SequentialBlocksCompressor::new(meta, self) |
| 77 |
} |
| 78 |
|
| 79 |
#[cfg(feature = "rayon")] |
| 80 |
/// Obtain a new writer that can compress blocks to chunks on multiple threads, which are then passed to this writer. |
| 81 |
/// Returns none if the sequential compressor should be used instead (thread pool creation failure or too large performance overhead). |
| 82 |
fn parallel_blocks_compressor<'w>(&'w mut self, meta: &'w MetaData) -> Option<ParallelBlocksCompressor<'w, Self>> { |
| 83 |
ParallelBlocksCompressor::new(meta, self) |
| 84 |
} |
| 85 |
|
| 86 |
/// Compresses all blocks to the file. |
| 87 |
/// The index of the block must be in increasing line order within the header. |
| 88 |
/// Obtain iterator with `MetaData::collect_ordered_blocks(...)` or similar methods. |
| 89 |
fn compress_all_blocks_sequential(mut self, meta: &MetaData, blocks: impl Iterator<Item=(usize, UncompressedBlock)>) -> UnitResult { |
| 90 |
let mut writer = self.sequential_blocks_compressor(meta); |
| 91 |
|
| 92 |
// TODO check block order if line order is not unspecified! |
| 93 |
for (index_in_header_increasing_y, block) in blocks { |
| 94 |
writer.compress_block(index_in_header_increasing_y, block)?; |
| 95 |
} |
| 96 |
|
| 97 |
// TODO debug_assert_eq!(self.is_complete()); |
| 98 |
Ok(()) |
| 99 |
} |
| 100 |
|
| 101 |
#[cfg(feature = "rayon")] |
| 102 |
/// Compresses all blocks to the file. |
| 103 |
/// The index of the block must be in increasing line order within the header. |
| 104 |
/// Obtain iterator with `MetaData::collect_ordered_blocks(...)` or similar methods. |
| 105 |
/// Will fallback to sequential processing where threads are not available, or where it would not speed up the process. |
| 106 |
fn compress_all_blocks_parallel(mut self, meta: &MetaData, blocks: impl Iterator<Item=(usize, UncompressedBlock)>) -> UnitResult { |
| 107 |
let mut parallel_writer = match self.parallel_blocks_compressor(meta) { |
| 108 |
None => return self.compress_all_blocks_sequential(meta, blocks), |
| 109 |
Some(writer) => writer, |
| 110 |
}; |
| 111 |
|
| 112 |
// TODO check block order if line order is not unspecified! |
| 113 |
for (index_in_header_increasing_y, block) in blocks { |
| 114 |
parallel_writer.add_block_to_compression_queue(index_in_header_increasing_y, block)?; |
| 115 |
} |
| 116 |
|
| 117 |
// TODO debug_assert_eq!(self.is_complete()); |
| 118 |
Ok(()) |
| 119 |
} |
| 120 |
} |
| 121 |
|
| 122 |
|
| 123 |
impl<W> ChunksWriter for ChunkWriter<W> where W: Write + Seek { |
| 124 |
|
| 125 |
/// The total number of chunks that the complete file will contain. |
| 126 |
fn total_chunks_count(&self) -> usize { self.chunk_count } |
| 127 |
|
| 128 |
/// Any more calls will result in an error and have no effect. |
| 129 |
/// If writing results in an error, the file and the writer |
| 130 |
/// may remain in an invalid state and should not be used further. |
| 131 |
/// Errors when the chunk at this index was already written. |
| 132 |
fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult { |
| 133 |
let header_chunk_indices = &mut self.chunk_indices_increasing_y[chunk.layer_index]; |
| 134 |
|
| 135 |
if index_in_header_increasing_y >= header_chunk_indices.len() { |
| 136 |
return Err(Error::invalid("too large chunk index")); |
| 137 |
} |
| 138 |
|
| 139 |
let chunk_index_slot = &mut header_chunk_indices[index_in_header_increasing_y]; |
| 140 |
if *chunk_index_slot != 0 { |
| 141 |
return Err(Error::invalid(format!("chunk at index {} is already written", index_in_header_increasing_y))); |
| 142 |
} |
| 143 |
|
| 144 |
*chunk_index_slot = usize_to_u64(self.byte_writer.byte_position(), "seek position")?; |
| 145 |
chunk.write(&mut self.byte_writer, self.header_count)?; |
| 146 |
Ok(()) |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
impl<W> ChunkWriter<W> where W: Write + Seek { |
| 151 |
// -- the following functions are private, because they must be called in a strict order -- |
| 152 |
|
| 153 |
/// Writes the meta data and zeroed offset tables as a placeholder. |
| 154 |
fn new_for_buffered(buffered_byte_writer: W, headers: Headers, pedantic: bool) -> Result<(MetaData, Self)> { |
| 155 |
let mut write = Tracking::new(buffered_byte_writer); |
| 156 |
let requirements = MetaData::write_validating_to_buffered(&mut write, headers.as_slice(), pedantic)?; |
| 157 |
|
| 158 |
// TODO: use increasing line order where possible, but this requires us to know whether we want to be parallel right now |
| 159 |
/*// if non-parallel compression, we always use increasing order anyways |
| 160 |
if !parallel || !has_compression { |
| 161 |
for header in &mut headers { |
| 162 |
if header.line_order == LineOrder::Unspecified { |
| 163 |
header.line_order = LineOrder::Increasing; |
| 164 |
} |
| 165 |
} |
| 166 |
}*/ |
| 167 |
|
| 168 |
let offset_table_size: usize = headers.iter().map(|header| header.chunk_count).sum(); |
| 169 |
|
| 170 |
let offset_table_start_byte = write.byte_position(); |
| 171 |
let offset_table_end_byte = write.byte_position() + offset_table_size * u64::BYTE_SIZE; |
| 172 |
|
| 173 |
// skip offset tables, filling with 0, will be updated after the last chunk has been written |
| 174 |
write.seek_write_to(offset_table_end_byte)?; |
| 175 |
|
| 176 |
let header_count = headers.len(); |
| 177 |
let chunk_indices_increasing_y = headers.iter() |
| 178 |
.map(|header| vec![0_u64; header.chunk_count]).collect(); |
| 179 |
|
| 180 |
let meta_data = MetaData { requirements, headers }; |
| 181 |
|
| 182 |
Ok((meta_data, ChunkWriter { |
| 183 |
header_count, |
| 184 |
byte_writer: write, |
| 185 |
chunk_count: offset_table_size, |
| 186 |
chunk_indices_byte_location: offset_table_start_byte .. offset_table_end_byte, |
| 187 |
chunk_indices_increasing_y, |
| 188 |
})) |
| 189 |
} |
| 190 |
|
| 191 |
/// Seek back to the meta data, write offset tables, and flush the byte writer. |
| 192 |
/// Leaves the writer seeked to the middle of the file, therefore we drop it. |
| 193 |
fn complete_meta_data(mut self) -> UnitResult { |
| 194 |
if self.chunk_indices_increasing_y.iter().flatten().any(|&index| index == 0) { |
| 195 |
return Err(Error::invalid("some chunks are not written yet")) |
| 196 |
} |
| 197 |
|
| 198 |
// write all offset tables |
| 199 |
debug_assert_ne!(self.byte_writer.byte_position(), self.chunk_indices_byte_location.end, "offset table has already been updated"); |
| 200 |
self.byte_writer.seek_write_to(self.chunk_indices_byte_location.start)?; |
| 201 |
|
| 202 |
for table in self.chunk_indices_increasing_y { |
| 203 |
u64::write_slice_le(&mut self.byte_writer, table.as_slice())?; |
| 204 |
} |
| 205 |
|
| 206 |
self.byte_writer.flush()?; // make sure we catch all (possibly delayed) io errors before returning |
| 207 |
Ok(()) |
| 208 |
} |
| 209 |
|
| 210 |
} |
| 211 |
|
| 212 |
|
| 213 |
impl<'w, W, F> ChunksWriter for OnProgressChunkWriter<'w, W, F> where W: 'w + ChunksWriter, F: FnMut(f64) { |
| 214 |
fn total_chunks_count(&self) -> usize { |
| 215 |
self.chunk_writer.total_chunks_count() |
| 216 |
} |
| 217 |
|
| 218 |
fn write_chunk(&mut self, index_in_header_increasing_y: usize, chunk: Chunk) -> UnitResult { |
| 219 |
let total_chunks = self.total_chunks_count(); |
| 220 |
let on_progress = &mut self.on_progress; |
| 221 |
|
| 222 |
// guarantee on_progress being called with 0 once |
| 223 |
if self.written_chunks == 0 { on_progress(0.0); } |
| 224 |
|
| 225 |
self.chunk_writer.write_chunk(index_in_header_increasing_y, chunk)?; |
| 226 |
|
| 227 |
self.written_chunks += 1; |
| 228 |
|
| 229 |
on_progress({ |
| 230 |
// guarantee finishing with progress 1.0 for last block at least once, float division might slightly differ from 1.0 |
| 231 |
if self.written_chunks == total_chunks { 1.0 } |
| 232 |
else { self.written_chunks as f64 / total_chunks as f64 } |
| 233 |
}); |
| 234 |
|
| 235 |
Ok(()) |
| 236 |
} |
| 237 |
} |
| 238 |
|
| 239 |
|
| 240 |
/// Write blocks that appear in any order and reorder them before writing. |
| 241 |
#[derive(Debug)] |
| 242 |
#[must_use] |
| 243 |
pub struct SortedBlocksWriter<'w, W> { |
| 244 |
chunk_writer: &'w mut W, |
| 245 |
pending_chunks: BTreeMap<usize, (usize, Chunk)>, |
| 246 |
unwritten_chunk_indices: Peekable<std::ops::Range<usize>>, |
| 247 |
requires_sorting: bool, // using this instead of Option, because of borrowing |
| 248 |
} |
| 249 |
|
| 250 |
|
| 251 |
impl<'w, W> SortedBlocksWriter<'w, W> where W: ChunksWriter { |
| 252 |
|
| 253 |
/// New sorting writer. Returns `None` if sorting is not required. |
| 254 |
pub fn new(meta_data: &MetaData, chunk_writer: &'w mut W) -> SortedBlocksWriter<'w, W> { |
| 255 |
let requires_sorting = meta_data.headers.iter() |
| 256 |
.any(|header| header.line_order != LineOrder::Unspecified); |
| 257 |
|
| 258 |
let total_chunk_count = chunk_writer.total_chunks_count(); |
| 259 |
|
| 260 |
SortedBlocksWriter { |
| 261 |
pending_chunks: BTreeMap::new(), |
| 262 |
unwritten_chunk_indices: (0 .. total_chunk_count).peekable(), |
| 263 |
requires_sorting, |
| 264 |
chunk_writer |
| 265 |
} |
| 266 |
} |
| 267 |
|
| 268 |
/// Write the chunk or stash it. In the closure, write all chunks that can be written now. |
| 269 |
pub fn write_or_stash_chunk(&mut self, chunk_index_in_file: usize, chunk_y_index: usize, chunk: Chunk) -> UnitResult { |
| 270 |
if self.requires_sorting.not() { |
| 271 |
return self.chunk_writer.write_chunk(chunk_y_index, chunk); |
| 272 |
} |
| 273 |
|
| 274 |
// write this chunk now if possible |
| 275 |
if self.unwritten_chunk_indices.peek() == Some(&chunk_index_in_file){ |
| 276 |
self.chunk_writer.write_chunk(chunk_y_index, chunk)?; |
| 277 |
self.unwritten_chunk_indices.next().expect("peeked chunk index is missing"); |
| 278 |
|
| 279 |
// write all pending blocks that are immediate successors of this block |
| 280 |
while let Some((next_chunk_y_index, next_chunk)) = self |
| 281 |
.unwritten_chunk_indices.peek().cloned() |
| 282 |
.and_then(|id| self.pending_chunks.remove(&id)) |
| 283 |
{ |
| 284 |
self.chunk_writer.write_chunk(next_chunk_y_index, next_chunk)?; |
| 285 |
self.unwritten_chunk_indices.next().expect("peeked chunk index is missing"); |
| 286 |
} |
| 287 |
} |
| 288 |
|
| 289 |
else { |
| 290 |
// the argument block is not to be written now, |
| 291 |
// and all the pending blocks are not next up either, |
| 292 |
// so just stash this block |
| 293 |
self.pending_chunks.insert(chunk_index_in_file, (chunk_y_index, chunk)); |
| 294 |
} |
| 295 |
|
| 296 |
Ok(()) |
| 297 |
} |
| 298 |
|
| 299 |
/// Where the chunks will be written to. |
| 300 |
pub fn inner_chunks_writer(&self) -> &W { |
| 301 |
&self.chunk_writer |
| 302 |
} |
| 303 |
} |
| 304 |
|
| 305 |
|
| 306 |
|
| 307 |
/// Compress blocks to a chunk writer in this thread. |
| 308 |
#[derive(Debug)] |
| 309 |
#[must_use] |
| 310 |
pub struct SequentialBlocksCompressor<'w, W> { |
| 311 |
meta: &'w MetaData, |
| 312 |
chunks_writer: &'w mut W, |
| 313 |
} |
| 314 |
|
| 315 |
impl<'w, W> SequentialBlocksCompressor<'w, W> where W: 'w + ChunksWriter { |
| 316 |
|
| 317 |
/// New blocks writer. |
| 318 |
pub fn new(meta: &'w MetaData, chunks_writer: &'w mut W) -> Self { Self { meta, chunks_writer, } } |
| 319 |
|
| 320 |
/// This is where the compressed blocks are written to. |
| 321 |
pub fn inner_chunks_writer(&'w self) -> &'w W { self.chunks_writer } |
| 322 |
|
| 323 |
/// Compress a single block immediately. The index of the block must be in increasing line order. |
| 324 |
pub fn compress_block(&mut self, index_in_header_increasing_y: usize, block: UncompressedBlock) -> UnitResult { |
| 325 |
self.chunks_writer.write_chunk( |
| 326 |
index_in_header_increasing_y, |
| 327 |
block.compress_to_chunk(&self.meta.headers)? |
| 328 |
) |
| 329 |
} |
| 330 |
} |
| 331 |
|
| 332 |
|
| 333 |
|
| 334 |
/// Compress blocks to a chunk writer with multiple threads. |
| 335 |
#[cfg(feature = "rayon")] |
| 336 |
#[derive(Debug)] |
| 337 |
#[must_use] |
| 338 |
pub struct ParallelBlocksCompressor<'w, W> { |
| 339 |
meta: &'w MetaData, |
| 340 |
sorted_writer: SortedBlocksWriter<'w, W>, |
| 341 |
|
| 342 |
sender: std::sync::mpsc::Sender<Result<(usize, usize, Chunk)>>, |
| 343 |
receiver: std::sync::mpsc::Receiver<Result<(usize, usize, Chunk)>>, |
| 344 |
pool: rayon_core::ThreadPool, |
| 345 |
|
| 346 |
currently_compressing_count: usize, |
| 347 |
written_chunk_count: usize, // used to check for last chunk |
| 348 |
max_threads: usize, |
| 349 |
next_incoming_chunk_index: usize, // used to remember original chunk order |
| 350 |
} |
| 351 |
|
| 352 |
#[cfg(feature = "rayon")] |
| 353 |
impl<'w, W> ParallelBlocksCompressor<'w, W> where W: 'w + ChunksWriter { |
| 354 |
|
| 355 |
/// New blocks writer. Returns none if sequential compression should be used. |
| 356 |
/// Use `new_with_thread_pool` to customize the threadpool. |
| 357 |
pub fn new(meta: &'w MetaData, chunks_writer: &'w mut W) -> Option<Self> { |
| 358 |
Self::new_with_thread_pool(meta, chunks_writer, ||{ |
| 359 |
rayon_core::ThreadPoolBuilder::new() |
| 360 |
.thread_name(|index| format!("OpenEXR Block Compressor Thread #{}", index)) |
| 361 |
.build() |
| 362 |
}) |
| 363 |
} |
| 364 |
|
| 365 |
/// New blocks writer. Returns none if sequential compression should be used. |
| 366 |
pub fn new_with_thread_pool<CreatePool>( |
| 367 |
meta: &'w MetaData, chunks_writer: &'w mut W, try_create_thread_pool: CreatePool) |
| 368 |
-> Option<Self> |
| 369 |
where CreatePool: FnOnce() -> std::result::Result<rayon_core::ThreadPool, rayon_core::ThreadPoolBuildError> |
| 370 |
{ |
| 371 |
use crate::compression::Compression; |
| 372 |
|
| 373 |
let is_entirely_uncompressed = meta.headers.iter() |
| 374 |
.all(|head|head.compression == Compression::Uncompressed); |
| 375 |
|
| 376 |
if is_entirely_uncompressed { |
| 377 |
return None; |
| 378 |
} |
| 379 |
|
| 380 |
// in case thread pool creation fails (for example on WASM currently), |
| 381 |
// we revert to sequential compression |
| 382 |
let pool = match try_create_thread_pool() { |
| 383 |
Ok(pool) => pool, |
| 384 |
|
| 385 |
// TODO print warning? |
| 386 |
Err(_) => return None, |
| 387 |
}; |
| 388 |
|
| 389 |
let max_threads = pool.current_num_threads().max(1).min(chunks_writer.total_chunks_count()) + 2; // ca one block for each thread at all times |
| 390 |
let (send, recv) = std::sync::mpsc::channel(); // TODO bounded channel simplifies logic? |
| 391 |
|
| 392 |
Some(Self { |
| 393 |
sorted_writer: SortedBlocksWriter::new(meta, chunks_writer), |
| 394 |
next_incoming_chunk_index: 0, |
| 395 |
currently_compressing_count: 0, |
| 396 |
written_chunk_count: 0, |
| 397 |
sender: send, |
| 398 |
receiver: recv, |
| 399 |
max_threads, |
| 400 |
pool, |
| 401 |
meta, |
| 402 |
}) |
| 403 |
} |
| 404 |
|
| 405 |
/// This is where the compressed blocks are written to. |
| 406 |
pub fn inner_chunks_writer(&'w self) -> &'w W { self.sorted_writer.inner_chunks_writer() } |
| 407 |
|
| 408 |
// private, as may underflow counter in release mode |
| 409 |
fn write_next_queued_chunk(&mut self) -> UnitResult { |
| 410 |
debug_assert!(self.currently_compressing_count > 0, "cannot wait for chunks as there are none left"); |
| 411 |
|
| 412 |
let some_compressed_chunk = self.receiver.recv() |
| 413 |
.expect("cannot receive compressed block"); |
| 414 |
|
| 415 |
self.currently_compressing_count -= 1; |
| 416 |
let (chunk_file_index, chunk_y_index, chunk) = some_compressed_chunk?; |
| 417 |
self.sorted_writer.write_or_stash_chunk(chunk_file_index, chunk_y_index, chunk)?; |
| 418 |
|
| 419 |
self.written_chunk_count += 1; |
| 420 |
Ok(()) |
| 421 |
} |
| 422 |
|
| 423 |
/// Wait until all currently compressing chunks in the compressor have been written. |
| 424 |
pub fn write_all_queued_chunks(&mut self) -> UnitResult { |
| 425 |
while self.currently_compressing_count > 0 { |
| 426 |
self.write_next_queued_chunk()?; |
| 427 |
} |
| 428 |
|
| 429 |
debug_assert_eq!(self.currently_compressing_count, 0, "counter does not match block count"); |
| 430 |
Ok(()) |
| 431 |
} |
| 432 |
|
| 433 |
/// Add a single block to the compressor queue. The index of the block must be in increasing line order. |
| 434 |
/// When calling this function for the last block, this method waits until all the blocks have been written. |
| 435 |
/// This only works when you write as many blocks as the image expects, otherwise you can use `wait_for_all_remaining_chunks`. |
| 436 |
/// Waits for a block from the queue to be written, if the queue already has enough items. |
| 437 |
pub fn add_block_to_compression_queue(&mut self, index_in_header_increasing_y: usize, block: UncompressedBlock) -> UnitResult { |
| 438 |
|
| 439 |
// if pipe is full, block to wait for a slot to free up |
| 440 |
if self.currently_compressing_count >= self.max_threads { |
| 441 |
self.write_next_queued_chunk()?; |
| 442 |
} |
| 443 |
|
| 444 |
// add the argument chunk to the compression queueue |
| 445 |
let index_in_file = self.next_incoming_chunk_index; |
| 446 |
let sender = self.sender.clone(); |
| 447 |
let meta = self.meta.clone(); |
| 448 |
|
| 449 |
self.pool.spawn(move ||{ |
| 450 |
let compressed_or_err = block.compress_to_chunk(&meta.headers); |
| 451 |
|
| 452 |
// by now, decompressing could have failed in another thread. |
| 453 |
// the error is then already handled, so we simply |
| 454 |
// don't send the decompressed block and do nothing |
| 455 |
let _ = sender.send(compressed_or_err.map(move |compressed| (index_in_file, index_in_header_increasing_y, compressed))); |
| 456 |
}); |
| 457 |
|
| 458 |
self.currently_compressing_count += 1; |
| 459 |
self.next_incoming_chunk_index += 1; |
| 460 |
|
| 461 |
// if this is the last chunk, wait for all chunks to complete before returning |
| 462 |
if self.written_chunk_count + self.currently_compressing_count == self.inner_chunks_writer().total_chunks_count() { |
| 463 |
self.write_all_queued_chunks()?; |
| 464 |
debug_assert_eq!( |
| 465 |
self.written_chunk_count, self.inner_chunks_writer().total_chunks_count(), |
| 466 |
"written chunk count mismatch" |
| 467 |
); |
| 468 |
} |
| 469 |
|
| 470 |
|
| 471 |
Ok(()) |
| 472 |
} |
| 473 |
} |
| 474 |
|
| 475 |
|
| 476 |
|