Debian Codegraph @

lib/aviread/Cache.cpp

raw
1 #include "Cache.h"
2 #include "avm_cpuinfo.h"
3 #include "avm_output.h"
4 #include "utils.h"
5
6 #include <unistd.h>
7 #include <fcntl.h>
8 #include <sys/time.h>
9 #include <sys/stat.h>
10
11 #include <errno.h>
12 #include <string.h>
13 #include <stdio.h>
14
15 // limit the maximum size of each prefetched stream in bytes
16 #define STREAM_SIZE_LIMIT 3000000
17
18 #ifndef O_LARGEFILE
19 #define O_LARGEFILE 0
20 #endif
21
22 // Isn't this bug in NetBSD configuration - it should not have HAVE_LSEEK64
23 #ifdef __NetBSD__
24 #define lseek64 lseek
25 #else
26 #ifndef HAVE_LSEEK64
27 #define lseek64 lseek
28 #endif
29 #endif
30
31
32 // uncomment if you do not want to use threads for precaching
33 //#define NOTHREADS // do not commit with this define
34
35 AVM_BEGIN_NAMESPACE;
36
37 #define __MODULE__ "StreamCache"
38 //static float ttt = 0;
39 //static float ttt1 = 0;
40
41 Cache::Cache(size_t size)
42 :m_uiSize(size), thread(0), m_pPacket(0), m_iFd(-1), m_uiId(0),
43 cache_access(0), cache_right(0), cache_miss(0), m_bQuit(false),
44 #ifndef NOTHREADS
45 m_bThreaded(true) // standard operation mode
46 #else
47 m_bThreaded(false) // for debugging purpose only - never commit
48 #endif
49 {
50 }
51
52 Cache::~Cache()
53 {
54 mutex.Lock();
55 m_bQuit = true;
56 cond.Broadcast();
57 mutex.Unlock();
58 delete thread;
59 clear();
60 //printf("Cache Access:%d Missed:%d Right:%d\n", cache_access, cache_miss, cache_right);
61 if (cache_access != 0)
62 AVM_WRITE(__MODULE__, "Destroy... (Total accesses %d, hits %.2f%%, misses %.2f%%, errors %.2f%%)\n",
63 cache_access, 100. * double(cache_right - cache_miss) / cache_access,
64 100. * double(cache_miss) / cache_access,
65 100. * double(cache_access - cache_right) / cache_access);
66 //printf("TTTT %f %f\n", ttt, ttt1);
67 }
68
69 int Cache::addStream(uint_t id, const avm::vector<uint32_t>& table)
70 {
71 AVM_WRITE(__MODULE__, 3, "Adding stream, %" PRIsz " chunks\n", table.size());
72 Locker locker(mutex);
73 m_streams.push_back(StreamEntry(&table, 0, m_uiSize));
74 cond.Broadcast();
75
76 return 0;
77 }
78
79 // starts caching thread once we know file descriptor
80 int Cache::create(int fd)
81 {
82 m_iFd = fd;
83 AVM_WRITE(__MODULE__, 1, "Creating cache for file descriptor: %d\n", m_iFd);
84 if (m_streams.size() > 0)
85 {
86 if (m_bThreaded)
87 {
88 Locker locker(mutex);
89 thread = new PthreadTask(0, &startThreadfunc, this);
90 if (!thread)
91 return -1;
92 cond.Wait(mutex);
93 }
94 }
95 else
96 AVM_WRITE(__MODULE__, "WARNING: No stream for caching!\n");
97
98 return 0;
99 }
100
101 // return true if this stream should be waken and read new data
102 inline bool Cache::isCachable(const StreamEntry& stream, uint_t id) const
103 {
104 // trick to allow precaching even very large image sizes
105 //printf("ISCHACHE f:%d s:%d sum:%d l: %d ll: %d\n", stream.packets.full(), stream.packets.size(), stream.sum, stream.last, stream.table->size());
106 return ((stream.sum < STREAM_SIZE_LIMIT
107 // assuming id=0 is video stream
108 // uncompressed video could be really huge!
109 //|| ((id == 0) && stream.packets.size() < 3)
110 )
111 && stream.last < stream.table->size()
112 && !stream.packets.full()
113 && (stream.filling
114 || (stream.sum < STREAM_SIZE_LIMIT/2
115 && stream.packets.size() < m_uiSize/2)));
116 }
117
118 //
119 // currently preffered picking alghorithm
120 //
121 // seems to be having good seek strategy
122 uint_t Cache::pickChunk()
123 {
124 uint_t id = m_uiId;
125
126 do
127 {
128 StreamEntry& se = m_streams[id];
129 // determine next needed chunk in this stream
130 se.last = (se.packets.empty()) ?
131 se.position : se.packets.back()->GetPos() + 1;
132 //printf("Pick sid:%d pos:%d wants:%d size:%d\n", id, se.last, se.position, se.packets.size());
133
134 if (isCachable(se, id))
135 return id;
136 // try next stream
137 if (++id >= m_streams.size())
138 id = 0; // wrap around
139 }
140 while (id != m_uiId);
141
142 return WAIT; // nothing for caching found
143 }
144
145 // caching thread
146 void* Cache::threadfunc()
147 {
148 size_t r = 0;
149 mutex.Lock();
150 while (!m_bQuit)
151 {
152 m_uiId = pickChunk();
153 cond.Broadcast();
154 // allow waiting tasks to use cache during read or Wait()
155 if (m_uiId == WAIT)
156 {
157 m_uiId = 0;
158
159 // one could be trying to send signal to this thread
160 //AVM_WRITE(__MODULE__, 4, "full packets - waiting for read: s#1: %d s#2: %d\n",
161 // m_streams[0].packets.size(), m_streams[1].packets.size());
162 cond.Wait(mutex);
163 //AVM_WRITE(__MODULE__, 4, "full packets - waiting done\n");
164 continue;
165 }
166
167 StreamEntry& stream = m_streams[m_uiId];
168 #if 0
169 if (stream.packets.size() > 0)
170 {
171 if (r > 20000)
172 {
173 r = 0;
174 // don't stress hardrive too much
175 //printf("sleep %d\n", stream.packets.size());
176 //cond.Wait(mutex, 0.05);
177 continue; // recheck if the same chunk is still needed
178 }
179 }
180 #endif
181 //printf("read %d l:%d p:%d s:%d\n", m_uiId, stream.last, stream.position, stream.packets.size());
182 uint_t coffset = (*stream.table)[stream.last];
183 char bfr[8];
184 //ttt1 += avm_get_time_diff(avm_get_time_us(), sss);
185 //cond.Broadcast();
186 mutex.Unlock();
187 //int64_t sss = avm_get_time_us();
188 if (lseek64(m_iFd, coffset & ~1, SEEK_SET) == -1
189 || ::read(m_iFd, bfr, 8) != 8)
190 {
191 AVM_WRITE(__MODULE__, "WARNING: Offset %d unreachable! %s\n", coffset & ~1, strerror(errno));
192 mutex.Lock();
193 stream.error = stream.last;
194 cond.Broadcast();
195 cond.Wait(mutex);
196 continue;
197 }
198
199 uint_t ckid = avm_get_le32(bfr);
200 uint_t clen = avm_get_le32(bfr + 4);
201
202 //printf("READ - pos %d ckid:0x%x %d of:0x%x\n", stream.last, ckid, clen, coffset);
203 // get free buffer
204 if (clen > StreamPacket::MAX_PACKET_SIZE)
205 {
206 // this is wrong and should be replaced by a better code
207 AVM_WRITE(__MODULE__, "WARNING: Too large chunk %d\n", clen);
208 clen = 10000;
209 }
210 if ((m_pPacket = new StreamPacket(clen)))
211 {
212 m_pPacket->SetPos(stream.last);
213 //AVM_WRITE(__MODULE__, 4, "id: %d %d buffered: %d sum: %d - %d\n",
214 // m_uiId, stream.last, (stream.last - stream.position),
215 // m_streams[0].sum, m_streams[1].sum);
216
217 // cache might be read while this chunk is being filled
218 size_t rs = 0;
219 while (rs < m_pPacket->GetSize())
220 {
221 ssize_t rd = ::read(m_iFd, m_pPacket->GetData() + rs, m_pPacket->GetSize() - rs);
222 //printf("READ %d %d of %d\n", rd, rs, m_pPacket->size);
223 if (rd <= 0)
224 {
225 if (stream.error == stream.OK)
226 AVM_WRITE(__MODULE__, "WARNING: Offset %d short read (%" PRIsz " < %" PRIsz ")! %s\n",
227 coffset, rs, m_pPacket->GetSize(), (rd < 0) ? strerror(errno) : "");
228 break;
229 }
230 rs += rd;
231 }
232 r += rs;
233 mutex.Lock();
234 //printf("readsize %d %d\n", (int)rs, (int)m_pPacket->size);
235 //printf("memch: "); for (int i = 0; i < m_pPacket->size && i < 20; i++) printf(" 0x%02x", m_pPacket->memory[i]); printf("\n");
236
237 // check if we still want same buffer
238 if (rs != m_pPacket->GetSize())
239 {
240 stream.error = stream.last;
241 m_pPacket->Release();
242 m_pPacket = 0;
243 cond.Broadcast();
244 cond.Wait(mutex);
245 continue;
246 }
247 //printf("Stream sum %d %d %d\n", stream.sum, stream.last, stream.position);
248 if (stream.packets.empty() && stream.position != stream.last) {
249 //printf("CH****************************** %d %d\n", stream.position, stream.last);
250 m_pPacket->Release();
251 m_pPacket = 0;
252 continue;
253 }
254
255 m_pPacket->SetFlags((coffset & 1) ? AVIIF_KEYFRAME : 0);
256 stream.error = stream.OK;
257 //uint_t sum = 0; for (uint_t i = 0 ; i < m_pPacket->size; i++) sum += ((unsigned char*) m_pPacket->memory)[i]; printf("PACKETSUM %d pos: %d size: %d\n", sum, m_pPacket->position, m_pPacket->size);
258
259 stream.sum += rs;
260 stream.filling = !(stream.sum > STREAM_SIZE_LIMIT);
261 stream.packets.push(m_pPacket);
262 //AVM_WRITE(__MODULE__, 4,
263 // "--- id: %d pos: %d sum: %d size: %d filling: %d\n",
264 // m_uiId, m_pPacket->position, stream.sum, m_pPacket->size, stream.filling);
265 m_pPacket = 0;
266 }
267 }
268
269 mutex.Unlock();
270 return 0;
271 }
272
273 // called by stream reader - most of the time this read should
274 // be satisfied from already precached chunks
275 StreamPacket* Cache::readPacket(uint_t id, framepos_t position)
276 {
277 //AVM_WRITE(__MODULE__, 4, "Cache: read(id %d, pos %d)\n", id, position);
278 ssize_t rsize = 1;
279 cache_access++;
280 if (id >= m_streams.size()) {
281 AVM_WRITE(__MODULE__, 1, "stream:%d out ouf bounds (%" PRIsz ")\n",
282 id, m_streams.size());
283 return 0;
284 }
285
286 StreamEntry& stream = m_streams[id];
287 if (position >= stream.table->size()) {
288 AVM_WRITE(__MODULE__, 1, "too large stream:%d pos: %d (of %" PRIsz ")\n",
289 id, position, stream.table->size());
290 return 0;
291 }
292
293 if (!m_bThreaded)
294 {
295 // code path for single threaded reading
296 //int64_t sss = avm_get_time_us();
297 Locker locker(mutex);
298 char bfr[8];
299 if (lseek64(m_iFd, (*stream.table)[position] & ~1, SEEK_SET) == -1
300 || ::read(m_iFd, bfr, 8) != 8) {
301 AVM_WRITE(__MODULE__, "WARNING: Read error\n");
302 return 0;
303 }
304
305 uint_t ckid = avm_get_le32(bfr);
306 uint_t clen = avm_get_le32(bfr + 4);
307 if (clen > StreamPacket::MAX_PACKET_SIZE)
308 {
309 AVM_WRITE(__MODULE__, "WARNING: Too large chunk %d\n", clen);
310 clen = StreamPacket::MAX_PACKET_SIZE;
311 }
312
313 StreamPacket* p;
314 if ((p = new StreamPacket(clen)))
315 {
316 if (p->GetSize() > 0)
317 {
318 // FIXME - this isn't going to give good result
319 rsize = ::read(m_iFd, p->GetData(), p->GetSize());
320 //printf("read_packet: id:%x len:%d rsize:%d %d m:%x\n", ckid, clen, rsize, p->size, *(int*)(p->memory));
321 if (rsize <= 0)
322 {
323 p->Release();
324 return 0;
325 }
326 }
327 p->SetFlags((*stream.table)[position] & 1 ? AVIIF_KEYFRAME : 0);
328 p->SetPos(position);
329 //ttt1 += avm_get_time_diff(avm_get_time_us(), sss);
330 }
331 return p;
332 }
333
334 mutex.Lock();
335 //while (stream.actual != position || stream.packets.size() == 0)
336 StreamPacket* p = 0;
337 while (!m_bQuit)
338 {
339 //printf("STREAMPOS:%d sp:%d id:%d ss:%d\n", position, stream.position, id, stream.packets.size());
340 if (!stream.packets.empty())
341 {
342 p = stream.packets.front();
343 stream.packets.pop();
344 stream.sum -= p->GetSize();
345 if (p->GetPos() == position)
346 {
347 //AVM_WRITE(__MODULE__, 4, "id: %d bsize: %d memory: %p pp: %d\n",
348 // id, stream.packets.size(), p->memory, p->position);
349 cache_right++;
350 break;
351 }
352 //AVM_WRITE(__MODULE__, 4, "position: 0x%x want: 0x%x\n", p->position, position);
353 // remove this chunk
354 //printf("delete chunkd %d (wants %d)\n", p->position, position);
355 p->Release();
356 p = 0;
357 continue;
358 }
359 if (stream.error == position) {
360 //printf("READERROR e:%d p:%d pl:%d\n", stream.error, position, stream.last);
361 break;
362 }
363 cache_miss++;
364 rsize = 0;
365 m_uiId = id;
366 stream.position = position;
367
368 //AVM_WRITE(__MODULE__, 4, "--- actual: %d size: %d\n", id, stream.packets.size());
369 //int64_t w = avm_get_time_us();
370 //printf("ToWait read sid:%d pos:%d size:%d\n", id, position, stream.packets.size());
371 cond.Broadcast();
372 cond.Wait(mutex);
373 //printf("----------- DoneWait read size:%d\n", stream.packets.size());
374 //ttt += avm_get_time_diff(avm_get_time_us(), w);
375 //AVM_WRITE(__MODULE__, 4, "--- actual: %d done - size: %d\n", id, stream.packets.size());
376 }
377
378 if (stream.packets.size() < (CACHE_SIZE / 2))
379 cond.Broadcast(); // wakeup only when buffers are getting low...
380 mutex.Unlock();
381 //printf("RETURN packet %p\n", p);
382 return p;
383 }
384
385 int Cache::clear()
386 {
387 AVM_WRITE(__MODULE__, 4, "Cleared\n");
388
389 mutex.Lock();
390 for (unsigned i = 0; i < m_streams.size(); i++)
391 {
392 StreamEntry& stream = m_streams[i];
393 while (stream.packets.size())
394 {
395 StreamPacket* r = stream.packets.front();
396 stream.packets.pop();
397 r->Release();
398 }
399 stream.sum = 0;
400 stream.position = 0;
401 }
402 m_uiId = 0;
403 cond.Broadcast();
404 mutex.Unlock();
405
406 return 0;
407 }
408
409 double Cache::getSize()
410 {
411 /*
412 int status=0;
413 for(int i=0; i<m_uiSize; i++)
414 if(req_buf[i].st==req::BUFFER_READY)status++;
415 return (double)status/m_uiSize;
416 */
417 return 1.;
418 }
419
420 void* Cache::startThreadfunc(void* arg)
421 {
422 Cache* c = (Cache*)arg;
423 c->mutex.Lock();
424 c->cond.Broadcast();
425 c->mutex.Unlock();
426 return c->threadfunc();
427 }
428
429
430 /*************************************************************/
431
432 const uint_t InputStream::BFRSIZE = 1024;
433
434 InputStream::InputStream()
435 : m_iFd(-1), m_pCache(0), m_bEof(false), m_pBuffer(0)
436 {
437 }
438
439 InputStream::~InputStream()
440 {
441 close();
442 delete[] m_pBuffer;
443 }
444
445 int InputStream::open(const char *pszFile)
446 {
447 if (!m_pBuffer && !(m_pBuffer = new uint8_t[BFRSIZE]))
448 return -1;
449
450 if ((m_iFd = ::open(pszFile, O_RDONLY | O_LARGEFILE)) < 0)
451 {
452 AVM_WRITE("InputStream", "Could not open file %s: %s\n", pszFile, strerror(errno));
453 return -1;
454 }
455
456 m_iPos = 0;
457 m_iBuffered = 0;
458 return m_iFd;
459 }
460
461 void InputStream::close()
462 {
463 delete m_pCache;
464 m_pCache = 0;
465 if (m_iFd >= 0)
466 ::close(m_iFd);
467 m_iFd = -1;
468 }
469
470 int InputStream::async()
471 {
472 if (!m_pCache)
473 m_pCache = new Cache();
474 return (m_pCache) ? m_pCache->create(m_iFd) : -1;
475 }
476
477 int InputStream::addStream(uint_t id, const avm::vector<uint32_t>& table)
478 {
479 if (!m_pCache)
480 m_pCache = new Cache();
481 return (m_pCache) ? m_pCache->addStream(id, table) : -1;
482 }
483
484 off_t InputStream::len() const
485 {
486 struct stat st;
487 fstat(m_iFd, &st);
488 return st.st_size;
489 }
490
491 off_t InputStream::seek(int64_t offset)
492 {
493 off_t cur = offset - pos();
494 if (cur == 0)
495 return offset;
496 if (cur <= m_iBuffered && -cur <= m_iPos)
497 return seekCur(cur);
498
499 m_iBuffered = m_iPos = 0;
500 m_bEof = false;
501 return lseek64(m_iFd, offset, SEEK_SET);
502 }
503
504 off_t InputStream::seekCur(int64_t offset)
505 {
506 m_bEof = false;
507 if (offset == 0)
508 return pos();
509
510 //printf("seekcur %lld p:%d b:%d\n", offset, m_iPos, m_iBuffered);
511
512 if (m_iPos >= m_iBuffered)
513 lseek64(m_iFd, offset, SEEK_CUR); // buffer empty
514 else if (offset > (m_iBuffered - m_iPos) || (m_iPos + offset) < 0)
515 {
516 // out of buffer
517 offset -= (m_iBuffered - m_iPos);
518 m_iBuffered = m_iPos = 0;
519 lseek64(m_iFd, offset, SEEK_CUR);
520 }
521 else {
522 //printf("seekcur %lld %d\n", offset, m_iPos);
523 m_iPos += offset;
524 }
525 //printf("seekcur return %d\n", pos());
526 return pos();
527 }
528
529 off_t InputStream::pos() const
530 {
531 off_t o = lseek64(m_iFd, 0, SEEK_CUR);
532 //printf("POS: %lld %d %d %lld\n", o, m_iPos, m_iBuffered, len());
533 if (m_iPos < m_iBuffered)
534 o -= (m_iBuffered - m_iPos);
535 else if (o > len())
536 o = len();
537 return o;
538 }
539
540 ssize_t InputStream::read(void* buffer, size_t size)
541 {
542 ssize_t r = 0;
543 if (m_iPos < m_iBuffered)
544 {
545 size_t copy = m_iBuffered - m_iPos;
546 if (size < copy)
547 copy = size;
548 memcpy(buffer, m_pBuffer + m_iPos, copy);
549 m_iPos += copy;
550 r = copy;
551 size -= copy;
552 buffer = (char*) buffer + copy;
553 }
554 if (size > 0)
555 {
556 ssize_t s = ::read(m_iFd, buffer, size);
557 if (s <= 0)
558 {
559 m_bEof = true;
560 s = 0;
561 }
562 r += s;
563 }
564
565 return r;
566 }
567
568 uint8_t InputStream::readByte()
569 {
570 #if 1
571 if (m_iPos >= m_iBuffered)
572 {
573 ssize_t r = ::read(m_iFd, m_pBuffer, BFRSIZE);
574 //printf("Read size %d %d\n", r, BFRSIZE);
575 if (r <= 0)
576 {
577 m_bEof = true;
578 return 0;
579 }
580 m_iBuffered = r;
581 m_iPos = 0;
582 }
583 return m_pBuffer[m_iPos++];
584 #else
585 uint8_t c;
586 if (::read(m_iFd, &c, 1) <= 0)
587 {
588 m_bEof = true;
589 c = 0;
590 }
591
592 return c;
593 #endif
594 }
595
596 #undef __MODULE__
597
598 AVM_END_NAMESPACE;