| 1 |
/* +------------------------------------------------------------------------+ |
| 2 |
| Mobile Robot Programming Toolkit (MRPT) | |
| 3 |
| https://www.mrpt.org/ | |
| 4 |
| | |
| 5 |
| Copyright (c) 2005-2026, Individual contributors, see AUTHORS file | |
| 6 |
| See: https://www.mrpt.org/Authors - All rights reserved. | |
| 7 |
| Released under BSD License. See: https://www.mrpt.org/License | |
| 8 |
+------------------------------------------------------------------------+ */ |
| 9 |
|
| 10 |
#if defined(__GNUC__) // Needed for ffmpeg headers. Only allowed here when not |
| 11 |
// using precomp. headers |
| 12 |
#define __STDC_CONSTANT_MACROS // Needed for having "UINT64_C" and so |
| 13 |
#endif |
| 14 |
// |
| 15 |
#include "hwdrivers-precomp.h" // Precompiled headers |
| 16 |
// |
| 17 |
#include <mrpt/config.h> |
| 18 |
|
| 19 |
#if MRPT_HAS_FFMPEG |
| 20 |
extern "C" |
| 21 |
{ |
| 22 |
#define _MSC_STDINT_H_ // We already have pstdint.h in MRPT |
| 23 |
#include <libavcodec/avcodec.h> |
| 24 |
#include <libavformat/avformat.h> |
| 25 |
#include <libavutil/imgutils.h> |
| 26 |
#include <libswscale/swscale.h> |
| 27 |
} |
| 28 |
#endif |
| 29 |
|
| 30 |
#include <mrpt/hwdrivers/CFFMPEG_InputStream.h> |
| 31 |
|
| 32 |
using namespace mrpt; |
| 33 |
using namespace mrpt::hwdrivers; |
| 34 |
|
| 35 |
// JLBC: This file takes portions of code from the example |
| 36 |
// "avcodec_sample.0.4.9.cpp" |
| 37 |
// |
| 38 |
// Minimum ffmpeg libs versions we want to support: |
| 39 |
// Ubuntu 16.04 LTS: avcodec 56.60.100, avutil 54.31.100, avformat 56.40.101 |
| 40 |
// Ubuntu 20.04 LTS: avcodec 58.54.100, avutil 56.31.100, avformat 58.29.100 |
| 41 |
// |
| 42 |
#if MRPT_HAS_FFMPEG |
| 43 |
namespace mrpt::hwdrivers |
| 44 |
{ |
| 45 |
// All context for ffmpeg: |
| 46 |
struct TFFMPEGContext |
| 47 |
{ |
| 48 |
AVFormatContext* pFormatCtx{nullptr}; |
| 49 |
int videoStream{0}; |
| 50 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 51 |
AVCodecParameters* pCodecPars{nullptr}; |
| 52 |
#endif |
| 53 |
AVCodec* pCodec{nullptr}; |
| 54 |
AVCodecContext* pCodecCtx{nullptr}; |
| 55 |
AVFrame* pFrame{nullptr}; |
| 56 |
AVFrame* pFrameRGB{nullptr}; |
| 57 |
SwsContext* img_convert_ctx{nullptr}; |
| 58 |
std::vector<uint8_t> buffer; |
| 59 |
}; |
| 60 |
} // namespace mrpt::hwdrivers |
| 61 |
#endif |
| 62 |
|
| 63 |
struct CFFMPEG_InputStream::Impl |
| 64 |
{ |
| 65 |
#if MRPT_HAS_FFMPEG |
| 66 |
TFFMPEGContext m_state; |
| 67 |
#endif |
| 68 |
}; |
| 69 |
|
| 70 |
/* -------------------------------------------------------- |
| 71 |
Ctor |
| 72 |
-------------------------------------------------------- */ |
| 73 |
CFFMPEG_InputStream::CFFMPEG_InputStream() |
| 74 |
#if MRPT_HAS_FFMPEG |
| 75 |
: |
| 76 |
m_impl(mrpt::make_impl<CFFMPEG_InputStream::Impl>()) |
| 77 |
{ |
| 78 |
// av_register_all() not needed in ffmpeg >=4.0 |
| 79 |
#if LIBAVFORMAT_VERSION_MAJOR < 58 |
| 80 |
// Register all formats and codecs |
| 81 |
av_register_all(); |
| 82 |
#endif |
| 83 |
} |
| 84 |
#else |
| 85 |
{ |
| 86 |
THROW_EXCEPTION("MRPT has been compiled without FFMPEG libraries."); |
| 87 |
} |
| 88 |
#endif |
| 89 |
|
| 90 |
/* -------------------------------------------------------- |
| 91 |
Dtor |
| 92 |
-------------------------------------------------------- */ |
| 93 |
CFFMPEG_InputStream::~CFFMPEG_InputStream() |
| 94 |
{ |
| 95 |
#if MRPT_HAS_FFMPEG |
| 96 |
// Close everything: |
| 97 |
this->close(); |
| 98 |
#endif |
| 99 |
} |
| 100 |
|
| 101 |
/* -------------------------------------------------------- |
| 102 |
isOpen |
| 103 |
-------------------------------------------------------- */ |
| 104 |
bool CFFMPEG_InputStream::isOpen() const |
| 105 |
{ |
| 106 |
#if MRPT_HAS_FFMPEG |
| 107 |
const TFFMPEGContext* ctx = &m_impl->m_state; |
| 108 |
return ctx->pFormatCtx != nullptr; |
| 109 |
#else |
| 110 |
return false; |
| 111 |
#endif |
| 112 |
} |
| 113 |
|
| 114 |
/* -------------------------------------------------------- |
| 115 |
openURL |
| 116 |
-------------------------------------------------------- */ |
| 117 |
bool CFFMPEG_InputStream::openURL( |
| 118 |
const std::string& url, |
| 119 |
bool grab_as_grayscale, |
| 120 |
bool verbose, |
| 121 |
const std::map<std::string, std::string>& optionsMap) |
| 122 |
{ |
| 123 |
#if MRPT_HAS_FFMPEG |
| 124 |
this->close(); // Close first |
| 125 |
|
| 126 |
TFFMPEGContext* ctx = &m_impl->m_state; |
| 127 |
|
| 128 |
this->m_url = url; |
| 129 |
this->m_grab_as_grayscale = grab_as_grayscale; |
| 130 |
|
| 131 |
AVDictionary* options = nullptr; // "create" an empty dictionary |
| 132 |
for (const auto& kv : optionsMap) av_dict_set(&options, kv.first.c_str(), kv.second.c_str(), 0); |
| 133 |
|
| 134 |
// Open video file |
| 135 |
if (avformat_open_input(&ctx->pFormatCtx, url.c_str(), nullptr, &options) != 0) |
| 136 |
{ |
| 137 |
ctx->pFormatCtx = nullptr; |
| 138 |
std::cerr << "[CFFMPEG_InputStream::openURL] Cannot open video: " << url << "\n"; |
| 139 |
return false; |
| 140 |
} |
| 141 |
|
| 142 |
// Retrieve stream information |
| 143 |
if (avformat_find_stream_info(ctx->pFormatCtx, nullptr) < 0) |
| 144 |
{ |
| 145 |
std::cerr << "[CFFMPEG_InputStream::openURL] Couldn't find stream " |
| 146 |
"information: " |
| 147 |
<< url << "\n"; |
| 148 |
return false; |
| 149 |
} |
| 150 |
|
| 151 |
// Dump information about file onto standard error |
| 152 |
if (verbose) |
| 153 |
{ |
| 154 |
av_dump_format(ctx->pFormatCtx, 0, url.c_str(), false); |
| 155 |
} |
| 156 |
|
| 157 |
// Find the first video stream |
| 158 |
ctx->videoStream = -1; |
| 159 |
for (unsigned int i = 0; i < ctx->pFormatCtx->nb_streams; i++) |
| 160 |
{ |
| 161 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 162 |
auto codecType = ctx->pFormatCtx->streams[i]->codecpar->codec_type; |
| 163 |
#else |
| 164 |
auto codecType = ctx->pFormatCtx->streams[i]->codec->codec_type; |
| 165 |
#endif |
| 166 |
if (codecType == AVMEDIA_TYPE_VIDEO) |
| 167 |
{ |
| 168 |
ctx->videoStream = (int)i; |
| 169 |
break; |
| 170 |
} |
| 171 |
} |
| 172 |
if (ctx->videoStream == -1) |
| 173 |
{ |
| 174 |
std::cerr << "[CFFMPEG_InputStream::openURL] Didn't find a video stream: " << url << "\n"; |
| 175 |
return false; |
| 176 |
} |
| 177 |
|
| 178 |
// Get a pointer to the codec context for the video stream |
| 179 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 180 |
ctx->pCodecPars = ctx->pFormatCtx->streams[ctx->videoStream]->codecpar; |
| 181 |
// Find the decoder for the video stream |
| 182 |
const AVCodec* codec = avcodec_find_decoder(ctx->pCodecPars->codec_id); |
| 183 |
#else |
| 184 |
ctx->pCodecCtx = ctx->pFormatCtx->streams[ctx->videoStream]->codec; |
| 185 |
// Find the decoder for the video stream |
| 186 |
codec = avcodec_find_decoder(ctx->pCodecCtx->codec_id); |
| 187 |
#endif |
| 188 |
if (codec == nullptr) |
| 189 |
{ |
| 190 |
std::cerr << "[CFFMPEG_InputStream::openURL] Codec not found: " << url << "\n"; |
| 191 |
return false; |
| 192 |
} |
| 193 |
|
| 194 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 195 |
ctx->pCodecCtx = avcodec_alloc_context3(nullptr /*ctx->pCodec*/); |
| 196 |
if (!ctx->pCodecCtx) |
| 197 |
{ |
| 198 |
std::cerr << "[CFFMPEG_InputStream::openURL] Cannot alloc avcodec " |
| 199 |
"context for: " |
| 200 |
<< url << "\n"; |
| 201 |
return false; |
| 202 |
} |
| 203 |
|
| 204 |
// Add stream parameters to context |
| 205 |
if (avcodec_parameters_to_context( |
| 206 |
ctx->pCodecCtx, ctx->pFormatCtx->streams[ctx->videoStream]->codecpar)) |
| 207 |
{ |
| 208 |
std::cerr << "[CFFMPEG_InputStream::openURL] Failed " |
| 209 |
"avcodec_parameters_to_context() for: " |
| 210 |
<< url << "\n"; |
| 211 |
return false; |
| 212 |
} |
| 213 |
|
| 214 |
// Make sure that Codecs are identical or avcodec_open2 fails. |
| 215 |
ctx->pCodecCtx->codec_id = codec->id; |
| 216 |
#endif |
| 217 |
|
| 218 |
// Open codec |
| 219 |
if (avcodec_open2(ctx->pCodecCtx, codec, nullptr) < 0) |
| 220 |
{ |
| 221 |
std::cerr << "[CFFMPEG_InputStream::openURL] avcodec_open2() failed for: " << url << "\n"; |
| 222 |
return false; |
| 223 |
} |
| 224 |
|
| 225 |
// Allocate video frame |
| 226 |
ctx->pFrame = av_frame_alloc(); |
| 227 |
// Allocate an AVFrame structure |
| 228 |
ctx->pFrameRGB = av_frame_alloc(); |
| 229 |
|
| 230 |
if (ctx->pFrameRGB == nullptr || ctx->pFrame == nullptr) |
| 231 |
{ |
| 232 |
std::cerr << "[CFFMPEG_InputStream::openURL] Could not alloc memory " |
| 233 |
"for frame buffers: " |
| 234 |
<< url << "\n"; |
| 235 |
return false; |
| 236 |
} |
| 237 |
|
| 238 |
// Determine required buffer size and allocate buffer |
| 239 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 240 |
const auto width = ctx->pCodecPars->width, height = ctx->pCodecPars->height; |
| 241 |
#else |
| 242 |
const auto width = ctx->pCodecCtx->width, height = ctx->pCodecCtx->height; |
| 243 |
#endif |
| 244 |
int numBytes = av_image_get_buffer_size( |
| 245 |
m_grab_as_grayscale ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_BGR24, width, height, 1); |
| 246 |
if (numBytes < 0) |
| 247 |
{ |
| 248 |
std::cerr << "[CFFMPEG_InputStream::openURL] av_image_get_buffer_size " |
| 249 |
"error code: " |
| 250 |
<< numBytes << "\n"; |
| 251 |
return false; |
| 252 |
} |
| 253 |
|
| 254 |
ctx->buffer.resize(numBytes); |
| 255 |
|
| 256 |
// Assign appropriate parts of buffer to image planes in pFrameRGB |
| 257 |
|
| 258 |
av_image_fill_arrays( |
| 259 |
ctx->pFrameRGB->data, ctx->pFrameRGB->linesize, &ctx->buffer[0], |
| 260 |
m_grab_as_grayscale ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_BGR24, width, height, 1); |
| 261 |
|
| 262 |
return true; // OK. |
| 263 |
#else |
| 264 |
return false; |
| 265 |
#endif |
| 266 |
} |
| 267 |
|
| 268 |
/* -------------------------------------------------------- |
| 269 |
close |
| 270 |
-------------------------------------------------------- */ |
| 271 |
void CFFMPEG_InputStream::close() |
| 272 |
{ |
| 273 |
#if MRPT_HAS_FFMPEG |
| 274 |
if (!this->isOpen()) return; |
| 275 |
|
| 276 |
TFFMPEGContext* ctx = &m_impl->m_state; |
| 277 |
|
| 278 |
// Close the codec |
| 279 |
if (ctx->pCodecCtx) |
| 280 |
{ |
| 281 |
avcodec_free_context(&ctx->pCodecCtx); |
| 282 |
} |
| 283 |
|
| 284 |
// Close the video file |
| 285 |
if (ctx->pFormatCtx) |
| 286 |
{ |
| 287 |
avformat_close_input(&ctx->pFormatCtx); |
| 288 |
ctx->pFormatCtx = nullptr; |
| 289 |
} |
| 290 |
|
| 291 |
// Free frames memory: |
| 292 |
ctx->buffer.clear(); |
| 293 |
|
| 294 |
if (ctx->pFrameRGB) |
| 295 |
{ |
| 296 |
av_frame_free(&ctx->pFrameRGB); |
| 297 |
ctx->pFrameRGB = nullptr; |
| 298 |
} |
| 299 |
if (ctx->pFrame) |
| 300 |
{ |
| 301 |
av_frame_free(&ctx->pFrame); |
| 302 |
ctx->pFrame = nullptr; |
| 303 |
} |
| 304 |
|
| 305 |
if (ctx->img_convert_ctx) |
| 306 |
{ |
| 307 |
sws_freeContext(ctx->img_convert_ctx); |
| 308 |
ctx->img_convert_ctx = nullptr; |
| 309 |
} |
| 310 |
|
| 311 |
#endif |
| 312 |
} |
| 313 |
|
| 314 |
/* -------------------------------------------------------- |
| 315 |
retrieveFrame |
| 316 |
-------------------------------------------------------- */ |
| 317 |
bool CFFMPEG_InputStream::retrieveFrame(mrpt::img::CImage& out_img) |
| 318 |
{ |
| 319 |
[[maybe_unused]] int64_t outPTS = 0; |
| 320 |
return retrieveFrame(out_img, outPTS); |
| 321 |
} |
| 322 |
|
| 323 |
bool CFFMPEG_InputStream::retrieveFrame(mrpt::img::CImage& out_img, int64_t& outPTS) |
| 324 |
{ |
| 325 |
#if MRPT_HAS_FFMPEG |
| 326 |
outPTS = 0; |
| 327 |
|
| 328 |
if (!this->isOpen()) return false; |
| 329 |
|
| 330 |
TFFMPEGContext* ctx = &m_impl->m_state; |
| 331 |
|
| 332 |
AVPacket packet; |
| 333 |
|
| 334 |
#if LIBAVFORMAT_VERSION_MAJOR < 57 |
| 335 |
int frameFinished; |
| 336 |
#endif |
| 337 |
|
| 338 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 339 |
const auto width = ctx->pCodecPars->width, height = ctx->pCodecPars->height; |
| 340 |
#else |
| 341 |
const auto width = ctx->pCodecCtx->width, height = ctx->pCodecCtx->height; |
| 342 |
#endif |
| 343 |
|
| 344 |
while (av_read_frame(ctx->pFormatCtx, &packet) >= 0) |
| 345 |
{ |
| 346 |
// Is this a packet from the video stream? |
| 347 |
if (packet.stream_index != ctx->videoStream) |
| 348 |
{ |
| 349 |
av_packet_unref(&packet); |
| 350 |
continue; |
| 351 |
} |
| 352 |
|
| 353 |
// Decode video frame |
| 354 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 355 |
int ret = avcodec_send_packet(ctx->pCodecCtx, &packet); |
| 356 |
if (ret < 0) |
| 357 |
{ |
| 358 |
std::cerr << "\n" |
| 359 |
<< "[CFFMPEG_InputStream] avcodec_send_packet error code=" << ret << "\n" |
| 360 |
<< "\n"; |
| 361 |
return false; |
| 362 |
} |
| 363 |
while (ret >= 0) |
| 364 |
{ |
| 365 |
ret = avcodec_receive_frame(ctx->pCodecCtx, ctx->pFrame); |
| 366 |
if (ret == AVERROR(EAGAIN)) continue; |
| 367 |
|
| 368 |
if (ret == AVERROR_EOF) |
| 369 |
return false; |
| 370 |
else if (ret < 0) |
| 371 |
{ |
| 372 |
std::cerr << "\n" |
| 373 |
<< "[CFFMPEG_InputStream] avcodec_receive_frame " |
| 374 |
"error code=" |
| 375 |
<< ret << "\n" |
| 376 |
<< "\n"; |
| 377 |
return false; |
| 378 |
} |
| 379 |
|
| 380 |
#else |
| 381 |
avcodec_decode_video2(ctx->pCodecCtx, ctx->pFrame, &frameFinished, &packet); |
| 382 |
if (!frameFinished) |
| 383 |
{ |
| 384 |
// Free the packet that was allocated by av_read_frame |
| 385 |
av_packet_unref(&packet); |
| 386 |
continue; |
| 387 |
} |
| 388 |
#endif |
| 389 |
// Convert the image from its native format to RGB: |
| 390 |
ctx->img_convert_ctx = sws_getCachedContext( |
| 391 |
ctx->img_convert_ctx, width, height, ctx->pCodecCtx->pix_fmt, width, height, |
| 392 |
m_grab_as_grayscale ? // BGR vs. RGB for OpenCV |
| 393 |
AV_PIX_FMT_GRAY8 |
| 394 |
: AV_PIX_FMT_BGR24, |
| 395 |
SWS_BICUBIC, nullptr, nullptr, nullptr); |
| 396 |
|
| 397 |
sws_scale( |
| 398 |
ctx->img_convert_ctx, ctx->pFrame->data, ctx->pFrame->linesize, 0, height, |
| 399 |
ctx->pFrameRGB->data, ctx->pFrameRGB->linesize); |
| 400 |
|
| 401 |
// std::cout << "[retrieveFrame] Generating image: " << |
| 402 |
// ctx->pCodecPars->width << "x" << ctx->pCodecPars->height |
| 403 |
// << "\n"; std::cout << " linsize: " << |
| 404 |
// ctx->pFrameRGB->linesize[0] << "\n"; |
| 405 |
|
| 406 |
if (ctx->pFrameRGB->linesize[0] != ((m_grab_as_grayscale ? 1 : 3) * width)) |
| 407 |
THROW_EXCEPTION("FIXME: linesize!=width case not handled yet."); |
| 408 |
|
| 409 |
out_img.loadFromMemoryBuffer(width, height, !m_grab_as_grayscale, ctx->pFrameRGB->data[0]); |
| 410 |
|
| 411 |
// Output frame timestamp: |
| 412 |
outPTS = ctx->pFrame->pts; |
| 413 |
|
| 414 |
// Free the packet that was allocated by av_read_frame |
| 415 |
av_packet_unref(&packet); |
| 416 |
return true; |
| 417 |
} |
| 418 |
#if LIBAVFORMAT_VERSION_MAJOR >= 57 |
| 419 |
} |
| 420 |
#endif |
| 421 |
return false; // Error reading/ EOF |
| 422 |
#else |
| 423 |
return false; |
| 424 |
#endif |
| 425 |
} |
| 426 |
|
| 427 |
/* -------------------------------------------------------- |
| 428 |
getVideoFPS |
| 429 |
-------------------------------------------------------- */ |
| 430 |
double CFFMPEG_InputStream::getVideoFPS() const |
| 431 |
{ |
| 432 |
#if MRPT_HAS_FFMPEG |
| 433 |
if (!this->isOpen()) return -1; |
| 434 |
|
| 435 |
const TFFMPEGContext* ctx = &m_impl->m_state; |
| 436 |
if (!ctx) return -1; |
| 437 |
if (!ctx->pCodecCtx) return -1; |
| 438 |
|
| 439 |
return double(ctx->pCodecCtx->framerate.num) / ctx->pCodecCtx->framerate.den; |
| 440 |
#else |
| 441 |
return false; |
| 442 |
#endif |
| 443 |
} |