#include "Cache.h" #include "avm_cpuinfo.h" #include "avm_output.h" #include "utils.h" #include #include #include #include #include #include #include // limit the maximum size of each prefetched stream in bytes #define STREAM_SIZE_LIMIT 3000000 #ifndef O_LARGEFILE #define O_LARGEFILE 0 #endif // Isn't this bug in NetBSD configuration - it should not have HAVE_LSEEK64 #ifdef __NetBSD__ #define lseek64 lseek #else #ifndef HAVE_LSEEK64 #define lseek64 lseek #endif #endif // uncomment if you do not want to use threads for precaching //#define NOTHREADS // do not commit with this define AVM_BEGIN_NAMESPACE; #define __MODULE__ "StreamCache" //static float ttt = 0; //static float ttt1 = 0; Cache::Cache(size_t size) :m_uiSize(size), thread(0), m_pPacket(0), m_iFd(-1), m_uiId(0), cache_access(0), cache_right(0), cache_miss(0), m_bQuit(false), #ifndef NOTHREADS m_bThreaded(true) // standard operation mode #else m_bThreaded(false) // for debugging purpose only - never commit #endif { } Cache::~Cache() { mutex.Lock(); m_bQuit = true; cond.Broadcast(); mutex.Unlock(); delete thread; clear(); //printf("Cache Access:%d Missed:%d Right:%d\n", cache_access, cache_miss, cache_right); if (cache_access != 0) AVM_WRITE(__MODULE__, "Destroy... (Total accesses %d, hits %.2f%%, misses %.2f%%, errors %.2f%%)\n", cache_access, 100. * double(cache_right - cache_miss) / cache_access, 100. * double(cache_miss) / cache_access, 100. * double(cache_access - cache_right) / cache_access); //printf("TTTT %f %f\n", ttt, ttt1); } int Cache::addStream(uint_t id, const avm::vector& table) { AVM_WRITE(__MODULE__, 3, "Adding stream, %" PRIsz " chunks\n", table.size()); Locker locker(mutex); m_streams.push_back(StreamEntry(&table, 0, m_uiSize)); cond.Broadcast(); return 0; } // starts caching thread once we know file descriptor int Cache::create(int fd) { m_iFd = fd; AVM_WRITE(__MODULE__, 1, "Creating cache for file descriptor: %d\n", m_iFd); if (m_streams.size() > 0) { if (m_bThreaded) { Locker locker(mutex); thread = new PthreadTask(0, &startThreadfunc, this); if (!thread) return -1; cond.Wait(mutex); } } else AVM_WRITE(__MODULE__, "WARNING: No stream for caching!\n"); return 0; } // return true if this stream should be waken and read new data inline bool Cache::isCachable(const StreamEntry& stream, uint_t id) const { // trick to allow precaching even very large image sizes //printf("ISCHACHE f:%d s:%d sum:%d l: %d ll: %d\n", stream.packets.full(), stream.packets.size(), stream.sum, stream.last, stream.table->size()); return ((stream.sum < STREAM_SIZE_LIMIT // assuming id=0 is video stream // uncompressed video could be really huge! //|| ((id == 0) && stream.packets.size() < 3) ) && stream.last < stream.table->size() && !stream.packets.full() && (stream.filling || (stream.sum < STREAM_SIZE_LIMIT/2 && stream.packets.size() < m_uiSize/2))); } // // currently preffered picking alghorithm // // seems to be having good seek strategy uint_t Cache::pickChunk() { uint_t id = m_uiId; do { StreamEntry& se = m_streams[id]; // determine next needed chunk in this stream se.last = (se.packets.empty()) ? se.position : se.packets.back()->GetPos() + 1; //printf("Pick sid:%d pos:%d wants:%d size:%d\n", id, se.last, se.position, se.packets.size()); if (isCachable(se, id)) return id; // try next stream if (++id >= m_streams.size()) id = 0; // wrap around } while (id != m_uiId); return WAIT; // nothing for caching found } // caching thread void* Cache::threadfunc() { size_t r = 0; mutex.Lock(); while (!m_bQuit) { m_uiId = pickChunk(); cond.Broadcast(); // allow waiting tasks to use cache during read or Wait() if (m_uiId == WAIT) { m_uiId = 0; // one could be trying to send signal to this thread //AVM_WRITE(__MODULE__, 4, "full packets - waiting for read: s#1: %d s#2: %d\n", // m_streams[0].packets.size(), m_streams[1].packets.size()); cond.Wait(mutex); //AVM_WRITE(__MODULE__, 4, "full packets - waiting done\n"); continue; } StreamEntry& stream = m_streams[m_uiId]; #if 0 if (stream.packets.size() > 0) { if (r > 20000) { r = 0; // don't stress hardrive too much //printf("sleep %d\n", stream.packets.size()); //cond.Wait(mutex, 0.05); continue; // recheck if the same chunk is still needed } } #endif //printf("read %d l:%d p:%d s:%d\n", m_uiId, stream.last, stream.position, stream.packets.size()); uint_t coffset = (*stream.table)[stream.last]; char bfr[8]; //ttt1 += avm_get_time_diff(avm_get_time_us(), sss); //cond.Broadcast(); mutex.Unlock(); //int64_t sss = avm_get_time_us(); if (lseek64(m_iFd, coffset & ~1, SEEK_SET) == -1 || ::read(m_iFd, bfr, 8) != 8) { AVM_WRITE(__MODULE__, "WARNING: Offset %d unreachable! %s\n", coffset & ~1, strerror(errno)); mutex.Lock(); stream.error = stream.last; cond.Broadcast(); cond.Wait(mutex); continue; } uint_t ckid = avm_get_le32(bfr); uint_t clen = avm_get_le32(bfr + 4); //printf("READ - pos %d ckid:0x%x %d of:0x%x\n", stream.last, ckid, clen, coffset); // get free buffer if (clen > StreamPacket::MAX_PACKET_SIZE) { // this is wrong and should be replaced by a better code AVM_WRITE(__MODULE__, "WARNING: Too large chunk %d\n", clen); clen = 10000; } if ((m_pPacket = new StreamPacket(clen))) { m_pPacket->SetPos(stream.last); //AVM_WRITE(__MODULE__, 4, "id: %d %d buffered: %d sum: %d - %d\n", // m_uiId, stream.last, (stream.last - stream.position), // m_streams[0].sum, m_streams[1].sum); // cache might be read while this chunk is being filled size_t rs = 0; while (rs < m_pPacket->GetSize()) { ssize_t rd = ::read(m_iFd, m_pPacket->GetData() + rs, m_pPacket->GetSize() - rs); //printf("READ %d %d of %d\n", rd, rs, m_pPacket->size); if (rd <= 0) { if (stream.error == stream.OK) AVM_WRITE(__MODULE__, "WARNING: Offset %d short read (%" PRIsz " < %" PRIsz ")! %s\n", coffset, rs, m_pPacket->GetSize(), (rd < 0) ? strerror(errno) : ""); break; } rs += rd; } r += rs; mutex.Lock(); //printf("readsize %d %d\n", (int)rs, (int)m_pPacket->size); //printf("memch: "); for (int i = 0; i < m_pPacket->size && i < 20; i++) printf(" 0x%02x", m_pPacket->memory[i]); printf("\n"); // check if we still want same buffer if (rs != m_pPacket->GetSize()) { stream.error = stream.last; m_pPacket->Release(); m_pPacket = 0; cond.Broadcast(); cond.Wait(mutex); continue; } //printf("Stream sum %d %d %d\n", stream.sum, stream.last, stream.position); if (stream.packets.empty() && stream.position != stream.last) { //printf("CH****************************** %d %d\n", stream.position, stream.last); m_pPacket->Release(); m_pPacket = 0; continue; } m_pPacket->SetFlags((coffset & 1) ? AVIIF_KEYFRAME : 0); stream.error = stream.OK; //uint_t sum = 0; for (uint_t i = 0 ; i < m_pPacket->size; i++) sum += ((unsigned char*) m_pPacket->memory)[i]; printf("PACKETSUM %d pos: %d size: %d\n", sum, m_pPacket->position, m_pPacket->size); stream.sum += rs; stream.filling = !(stream.sum > STREAM_SIZE_LIMIT); stream.packets.push(m_pPacket); //AVM_WRITE(__MODULE__, 4, // "--- id: %d pos: %d sum: %d size: %d filling: %d\n", // m_uiId, m_pPacket->position, stream.sum, m_pPacket->size, stream.filling); m_pPacket = 0; } } mutex.Unlock(); return 0; } // called by stream reader - most of the time this read should // be satisfied from already precached chunks StreamPacket* Cache::readPacket(uint_t id, framepos_t position) { //AVM_WRITE(__MODULE__, 4, "Cache: read(id %d, pos %d)\n", id, position); ssize_t rsize = 1; cache_access++; if (id >= m_streams.size()) { AVM_WRITE(__MODULE__, 1, "stream:%d out ouf bounds (%" PRIsz ")\n", id, m_streams.size()); return 0; } StreamEntry& stream = m_streams[id]; if (position >= stream.table->size()) { AVM_WRITE(__MODULE__, 1, "too large stream:%d pos: %d (of %" PRIsz ")\n", id, position, stream.table->size()); return 0; } if (!m_bThreaded) { // code path for single threaded reading //int64_t sss = avm_get_time_us(); Locker locker(mutex); char bfr[8]; if (lseek64(m_iFd, (*stream.table)[position] & ~1, SEEK_SET) == -1 || ::read(m_iFd, bfr, 8) != 8) { AVM_WRITE(__MODULE__, "WARNING: Read error\n"); return 0; } uint_t ckid = avm_get_le32(bfr); uint_t clen = avm_get_le32(bfr + 4); if (clen > StreamPacket::MAX_PACKET_SIZE) { AVM_WRITE(__MODULE__, "WARNING: Too large chunk %d\n", clen); clen = StreamPacket::MAX_PACKET_SIZE; } StreamPacket* p; if ((p = new StreamPacket(clen))) { if (p->GetSize() > 0) { // FIXME - this isn't going to give good result rsize = ::read(m_iFd, p->GetData(), p->GetSize()); //printf("read_packet: id:%x len:%d rsize:%d %d m:%x\n", ckid, clen, rsize, p->size, *(int*)(p->memory)); if (rsize <= 0) { p->Release(); return 0; } } p->SetFlags((*stream.table)[position] & 1 ? AVIIF_KEYFRAME : 0); p->SetPos(position); //ttt1 += avm_get_time_diff(avm_get_time_us(), sss); } return p; } mutex.Lock(); //while (stream.actual != position || stream.packets.size() == 0) StreamPacket* p = 0; while (!m_bQuit) { //printf("STREAMPOS:%d sp:%d id:%d ss:%d\n", position, stream.position, id, stream.packets.size()); if (!stream.packets.empty()) { p = stream.packets.front(); stream.packets.pop(); stream.sum -= p->GetSize(); if (p->GetPos() == position) { //AVM_WRITE(__MODULE__, 4, "id: %d bsize: %d memory: %p pp: %d\n", // id, stream.packets.size(), p->memory, p->position); cache_right++; break; } //AVM_WRITE(__MODULE__, 4, "position: 0x%x want: 0x%x\n", p->position, position); // remove this chunk //printf("delete chunkd %d (wants %d)\n", p->position, position); p->Release(); p = 0; continue; } if (stream.error == position) { //printf("READERROR e:%d p:%d pl:%d\n", stream.error, position, stream.last); break; } cache_miss++; rsize = 0; m_uiId = id; stream.position = position; //AVM_WRITE(__MODULE__, 4, "--- actual: %d size: %d\n", id, stream.packets.size()); //int64_t w = avm_get_time_us(); //printf("ToWait read sid:%d pos:%d size:%d\n", id, position, stream.packets.size()); cond.Broadcast(); cond.Wait(mutex); //printf("----------- DoneWait read size:%d\n", stream.packets.size()); //ttt += avm_get_time_diff(avm_get_time_us(), w); //AVM_WRITE(__MODULE__, 4, "--- actual: %d done - size: %d\n", id, stream.packets.size()); } if (stream.packets.size() < (CACHE_SIZE / 2)) cond.Broadcast(); // wakeup only when buffers are getting low... mutex.Unlock(); //printf("RETURN packet %p\n", p); return p; } int Cache::clear() { AVM_WRITE(__MODULE__, 4, "Cleared\n"); mutex.Lock(); for (unsigned i = 0; i < m_streams.size(); i++) { StreamEntry& stream = m_streams[i]; while (stream.packets.size()) { StreamPacket* r = stream.packets.front(); stream.packets.pop(); r->Release(); } stream.sum = 0; stream.position = 0; } m_uiId = 0; cond.Broadcast(); mutex.Unlock(); return 0; } double Cache::getSize() { /* int status=0; for(int i=0; imutex.Lock(); c->cond.Broadcast(); c->mutex.Unlock(); return c->threadfunc(); } /*************************************************************/ const uint_t InputStream::BFRSIZE = 1024; InputStream::InputStream() : m_iFd(-1), m_pCache(0), m_bEof(false), m_pBuffer(0) { } InputStream::~InputStream() { close(); delete[] m_pBuffer; } int InputStream::open(const char *pszFile) { if (!m_pBuffer && !(m_pBuffer = new uint8_t[BFRSIZE])) return -1; if ((m_iFd = ::open(pszFile, O_RDONLY | O_LARGEFILE)) < 0) { AVM_WRITE("InputStream", "Could not open file %s: %s\n", pszFile, strerror(errno)); return -1; } m_iPos = 0; m_iBuffered = 0; return m_iFd; } void InputStream::close() { delete m_pCache; m_pCache = 0; if (m_iFd >= 0) ::close(m_iFd); m_iFd = -1; } int InputStream::async() { if (!m_pCache) m_pCache = new Cache(); return (m_pCache) ? m_pCache->create(m_iFd) : -1; } int InputStream::addStream(uint_t id, const avm::vector& table) { if (!m_pCache) m_pCache = new Cache(); return (m_pCache) ? m_pCache->addStream(id, table) : -1; } off_t InputStream::len() const { struct stat st; fstat(m_iFd, &st); return st.st_size; } off_t InputStream::seek(int64_t offset) { off_t cur = offset - pos(); if (cur == 0) return offset; if (cur <= m_iBuffered && -cur <= m_iPos) return seekCur(cur); m_iBuffered = m_iPos = 0; m_bEof = false; return lseek64(m_iFd, offset, SEEK_SET); } off_t InputStream::seekCur(int64_t offset) { m_bEof = false; if (offset == 0) return pos(); //printf("seekcur %lld p:%d b:%d\n", offset, m_iPos, m_iBuffered); if (m_iPos >= m_iBuffered) lseek64(m_iFd, offset, SEEK_CUR); // buffer empty else if (offset > (m_iBuffered - m_iPos) || (m_iPos + offset) < 0) { // out of buffer offset -= (m_iBuffered - m_iPos); m_iBuffered = m_iPos = 0; lseek64(m_iFd, offset, SEEK_CUR); } else { //printf("seekcur %lld %d\n", offset, m_iPos); m_iPos += offset; } //printf("seekcur return %d\n", pos()); return pos(); } off_t InputStream::pos() const { off_t o = lseek64(m_iFd, 0, SEEK_CUR); //printf("POS: %lld %d %d %lld\n", o, m_iPos, m_iBuffered, len()); if (m_iPos < m_iBuffered) o -= (m_iBuffered - m_iPos); else if (o > len()) o = len(); return o; } ssize_t InputStream::read(void* buffer, size_t size) { ssize_t r = 0; if (m_iPos < m_iBuffered) { size_t copy = m_iBuffered - m_iPos; if (size < copy) copy = size; memcpy(buffer, m_pBuffer + m_iPos, copy); m_iPos += copy; r = copy; size -= copy; buffer = (char*) buffer + copy; } if (size > 0) { ssize_t s = ::read(m_iFd, buffer, size); if (s <= 0) { m_bEof = true; s = 0; } r += s; } return r; } uint8_t InputStream::readByte() { #if 1 if (m_iPos >= m_iBuffered) { ssize_t r = ::read(m_iFd, m_pBuffer, BFRSIZE); //printf("Read size %d %d\n", r, BFRSIZE); if (r <= 0) { m_bEof = true; return 0; } m_iBuffered = r; m_iPos = 0; } return m_pBuffer[m_iPos++]; #else uint8_t c; if (::read(m_iFd, &c, 1) <= 0) { m_bEof = true; c = 0; } return c; #endif } #undef __MODULE__ AVM_END_NAMESPACE;