Debian Codegraph @

src/bino.cpp

raw
1 /*
2 * This file is part of Bino, a 3D video player.
3 *
4 * Copyright (C) 2022, 2023, 2024, 2025
5 * Martin Lambers <[email protected]>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include <QFont>
22 #include <QFontMetrics>
23 #include <QTextLayout>
24 #include <QPainter>
25
26 #include "bino.hpp"
27 #include "log.hpp"
28 #include "tools.hpp"
29 #include "digestiblemedia.hpp"
30 #include "metadata.hpp"
31
32
33 static Bino* binoSingleton = nullptr;
34
35 Bino::Bino(ScreenType screenType, const Screen& screen, bool swapEyes) :
36 _wantExit(false),
37 _videoSink(nullptr),
38 _audioOutput(nullptr),
39 _player(nullptr),
40 _audioInput(nullptr),
41 _videoInput(nullptr),
42 _screenInput(nullptr),
43 _windowInput(nullptr),
44 _captureSession(nullptr),
45 _lastFrameInputMode(Input_Unknown),
46 _lastFrameSurroundMode(Surround_Unknown),
47 _screenType(screenType),
48 _screen(screen),
49 _frameIsNew(false),
50 _frameWasSerialized(true),
51 _swapEyes(swapEyes)
52 {
53 Q_ASSERT(!binoSingleton);
54 binoSingleton = this;
55 }
56
57 Bino::~Bino()
58 {
59 delete _videoSink;
60 delete _audioOutput;
61 delete _player;
62 delete _audioInput;
63 delete _videoInput;
64 delete _screenInput;
65 delete _windowInput;
66 delete _captureSession;
67 binoSingleton = nullptr;
68 }
69
70 Bino* Bino::instance()
71 {
72 return binoSingleton;
73 }
74
75 void Bino::initializeOutput(const QAudioDevice& audioOutputDevice)
76 {
77 _videoSink = new VideoSink(&_frame, &_extFrame, &_frameIsNew);
78 connect(_videoSink, &VideoSink::newVideoFrame, [=]() { emit newVideoFrame(); });
79 connect(_videoSink, &VideoSink::newVideoFrame, [=]() { _frameWasSerialized = false; });
80 _audioOutput = new QAudioOutput;
81 _audioOutput->setDevice(audioOutputDevice);
82 }
83
84 void Bino::startPlaylistMode()
85 {
86 if (playlistMode()) {
87 return;
88 }
89 if (captureMode()) {
90 stopCaptureMode();
91 }
92
93 connect(Playlist::instance(), SIGNAL(mediaChanged(PlaylistEntry)), this, SLOT(mediaChanged(PlaylistEntry)));
94
95 _player = new QMediaPlayer;
96 _player->setVideoOutput(_videoSink);
97 _player->setAudioOutput(_audioOutput);
98 _player->connect(_player, &QMediaPlayer::errorOccurred,
99 [=](QMediaPlayer::Error /* error */, const QString& errorString) {
100 LOG_WARNING("%s", qPrintable(tr("Media player error: %1").arg(errorString)));
101 });
102 _player->connect(_player, &QMediaPlayer::playbackStateChanged,
103 [=](QMediaPlayer::PlaybackState state) {
104 LOG_DEBUG("Playback state changed to %s",
105 state == QMediaPlayer::StoppedState ? "stopped"
106 : state == QMediaPlayer::PlayingState ? "playing"
107 : state == QMediaPlayer::PausedState ? "paused"
108 : "unknown");
109 if (state == QMediaPlayer::StoppedState)
110 Playlist::instance()->mediaEnded();
111 });
112
113 emit stateChanged();
114 }
115
116 void Bino::stopPlaylistMode()
117 {
118 if (_player) {
119 delete _player;
120 _player = nullptr;
121 emit stateChanged();
122 }
123 }
124
125 void Bino::startCaptureMode(bool withAudioInput, const QAudioDevice& audioInputDevice, InputMode inputMode)
126 {
127 if (playlistMode())
128 stopPlaylistMode();
129 if (captureMode())
130 stopCaptureMode();
131
132 _captureSession = new QMediaCaptureSession;
133 _captureSession->setAudioOutput(_audioOutput);
134 _captureSession->setVideoSink(_videoSink);
135 if (withAudioInput) {
136 _audioInput = new QAudioInput;
137 _audioInput->setDevice(audioInputDevice);
138 _captureSession->setAudioInput(_audioInput);
139 }
140 setInputMode(inputMode);
141 }
142
143 void Bino::startCaptureModeCamera(
144 bool withAudioInput,
145 const QAudioDevice& audioInputDevice,
146 const QCameraDevice& videoInputDevice,
147 InputMode inputMode)
148 {
149 startCaptureMode(withAudioInput, audioInputDevice, inputMode);
150 _videoInput = new QCamera;
151 _videoInput->setCameraDevice(videoInputDevice);
152 _captureSession->setCamera(_videoInput);
153 _videoInput->setActive(true);
154 emit stateChanged();
155 }
156
157 void Bino::startCaptureModeScreen(
158 bool withAudioInput,
159 const QAudioDevice& audioInputDevice,
160 QScreen* screenInputDevice,
161 InputMode inputMode)
162 {
163 startCaptureMode(withAudioInput, audioInputDevice, inputMode);
164 _screenInput = new QScreenCapture;
165 _screenInput->setScreen(screenInputDevice);
166 _captureSession->setScreenCapture(_screenInput);
167 _screenInput->setActive(true);
168 emit stateChanged();
169 }
170
171 void Bino::startCaptureModeWindow(
172 bool withAudioInput,
173 const QAudioDevice& audioInputDevice,
174 const QCapturableWindow& windowInputDevice,
175 InputMode inputMode)
176 {
177 startCaptureMode(withAudioInput, audioInputDevice, inputMode);
178 _windowInput = new QWindowCapture;
179 _windowInput->setWindow(windowInputDevice);
180 _captureSession->setWindowCapture(_windowInput);
181 _windowInput->setActive(true);
182 emit stateChanged();
183 }
184
185 void Bino::stopCaptureMode()
186 {
187 if (_captureSession) {
188 delete _captureSession;
189 _captureSession = nullptr;
190 delete _videoInput;
191 _videoInput = nullptr;
192 delete _screenInput;
193 _screenInput = nullptr;
194 delete _windowInput;
195 _windowInput = nullptr;
196 if (_audioInput) {
197 delete _audioInput;
198 _audioInput = nullptr;
199 }
200 emit stateChanged();
201 }
202 }
203
204 bool Bino::playlistMode() const
205 {
206 return _player;
207 }
208
209 bool Bino::captureMode() const
210 {
211 return _captureSession;
212 }
213
214 void Bino::mediaChanged(PlaylistEntry entry)
215 {
216 if (!playlistMode())
217 return;
218 if (entry.noMedia()) {
219 _player->stop();
220 } else {
221 // Get meta data
222 MetaData metaData;
223 metaData.detectCached(entry.url);
224 // Special handling of files that cannot be digested by QtMultimedia directly
225 QUrl digestibleUrl = digestibleMediaUrl(entry.url);
226 // Set new source
227 _player->setSource(digestibleUrl);
228 if (entry.videoTrack >= 0) {
229 _player->setActiveVideoTrack(entry.videoTrack);
230 }
231 if (entry.audioTrack >= 0) {
232 _player->setActiveAudioTrack(entry.audioTrack);
233 } else if (Playlist::instance()->preferredAudio() != QLocale::AnyLanguage) {
234 int audioTrack = -1;
235 for (int i = 0; i < int(metaData.audioTracks.length()); i++) {
236 QLocale audioLanguage = metaData.audioTracks[i].value(QMediaMetaData::Language).toLocale();
237 if (audioLanguage == Playlist::instance()->preferredAudio()) {
238 audioTrack = i;
239 break;
240 }
241 }
242 if (audioTrack >= 0) {
243 _player->setActiveAudioTrack(entry.audioTrack);
244 }
245 }
246 if (entry.subtitleTrack >= 0) {
247 _player->setActiveSubtitleTrack(entry.subtitleTrack);
248 } else if (entry.subtitleTrack == PlaylistEntry::NoTrack) {
249 // do nothing
250 } else if (metaData.subtitleTracks.size() > 0 && Playlist::instance()->wantSubtitle()) {
251 int subtitleTrack = 0;
252 for (int i = 0; i < int(metaData.subtitleTracks.length()); i++) {
253 QLocale subtitleLanguage = metaData.subtitleTracks[i].value(QMediaMetaData::Language).toLocale();
254 if (subtitleLanguage == Playlist::instance()->preferredSubtitle()) {
255 subtitleTrack = i;
256 break;
257 }
258 }
259 _player->setActiveSubtitleTrack(subtitleTrack);
260 }
261 _player->play();
262 _videoSink->newPlaylistEntry(entry, metaData);
263 }
264 emit stateChanged();
265 }
266
267 void Bino::quit()
268 {
269 stop();
270 _wantExit = true;
271 emit wantQuit();
272 }
273
274 void Bino::seek(qint64 milliseconds)
275 {
276 if (!playlistMode())
277 return;
278 _player->setPosition(_player->position() + milliseconds);
279 }
280
281 void Bino::setPosition(float pos)
282 {
283 if (!playlistMode())
284 return;
285 _player->setPosition(pos * _player->duration());
286 }
287
288 void Bino::togglePause()
289 {
290 if (!playlistMode())
291 return;
292 if (_player->playbackState() == QMediaPlayer::PlayingState) {
293 _player->pause();
294 emit stateChanged();
295 } else if (_player->playbackState() == QMediaPlayer::PausedState) {
296 _player->play();
297 emit stateChanged();
298 }
299 }
300
301 void Bino::pause()
302 {
303 if (!playlistMode())
304 return;
305 if (_player->playbackState() == QMediaPlayer::PlayingState) {
306 _player->pause();
307 emit stateChanged();
308 }
309 }
310
311 void Bino::play()
312 {
313 if (!playlistMode())
314 return;
315 if (_player->playbackState() != QMediaPlayer::PlayingState) {
316 _player->play();
317 emit stateChanged();
318 }
319 }
320
321 void Bino::setMute(bool m)
322 {
323 bool isMuted = _player->audioOutput()->isMuted();
324 if (m != isMuted) {
325 _audioOutput->setMuted(m);
326 emit stateChanged();
327 }
328 }
329
330 void Bino::toggleMute()
331 {
332 _audioOutput->setMuted(!_player->audioOutput()->isMuted());
333 emit stateChanged();
334 }
335
336 void Bino::setVolume(float vol)
337 {
338 _audioOutput->setVolume(vol);
339 }
340
341 void Bino::changeVolume(float offset)
342 {
343 _audioOutput->setVolume(_player->audioOutput()->volume() + offset);
344 }
345
346 void Bino::stop()
347 {
348 if (!playlistMode())
349 return;
350 if (_player->playbackState() == QMediaPlayer::StoppedState) {
351 _player->stop();
352 emit stateChanged();
353 }
354 }
355
356 void Bino::setSwapEyes(bool s)
357 {
358 if (_swapEyes != s) {
359 _swapEyes = s;
360 emit stateChanged();
361 }
362 }
363
364 void Bino::toggleSwapEyes()
365 {
366 _swapEyes = !_swapEyes;
367 emit stateChanged();
368 }
369
370 void Bino::setVideoTrack(int i)
371 {
372 LOG_DEBUG("changing video track to %d", i);
373 if (playlistMode()) {
374 _player->setActiveVideoTrack(i);
375 emit stateChanged();
376 }
377 }
378
379 void Bino::setAudioTrack(int i)
380 {
381 LOG_DEBUG("changing audio track to %d", i);
382 if (playlistMode()) {
383 _player->setActiveAudioTrack(i);
384 emit stateChanged();
385 }
386 }
387
388 void Bino::setSubtitleTrack(int i)
389 {
390 LOG_DEBUG("changing subtitle track to %d", i);
391 if (playlistMode()) {
392 _player->setActiveSubtitleTrack(i);
393 emit stateChanged();
394 }
395 }
396
397 void Bino::setInputMode(InputMode mode)
398 {
399 _videoSink->inputMode = mode;
400 _frame.inputMode = mode;
401 _frame.reUpdate();
402 _frameIsNew = true;
403 LOG_DEBUG("setting input mode to %s", inputModeToString(mode));
404 }
405
406 void Bino::setSurroundMode(SurroundMode mode)
407 {
408 _videoSink->surroundMode = mode;
409 _frame.surroundMode = mode;
410 _frame.reUpdate();
411 _frameIsNew = true;
412 LOG_DEBUG("setting surround mode to %s", surroundModeToString(mode));
413 }
414
415 bool Bino::swapEyes() const
416 {
417 return _swapEyes;
418 }
419
420 bool Bino::muted() const
421 {
422 return _audioOutput->isMuted();
423 }
424
425 bool Bino::paused() const
426 {
427 return (playlistMode() && _player->playbackState() == QMediaPlayer::PausedState);
428 }
429
430 bool Bino::playing() const
431 {
432 return (playlistMode() && _player->playbackState() == QMediaPlayer::PlayingState);
433 }
434
435 bool Bino::stopped() const
436 {
437 return (playlistMode() && _player->playbackState() == QMediaPlayer::StoppedState);
438 }
439
440 QUrl Bino::url() const
441 {
442 QUrl url;
443 if (playing() || paused())
444 url = _player->source();
445 return url;
446 }
447
448 int Bino::videoTrack() const
449 {
450 int t = -1;
451 if (captureMode())
452 t = 0;
453 else if (playlistMode())
454 t = _player->activeVideoTrack();
455 return t;
456 }
457
458 int Bino::audioTrack() const
459 {
460 int t = -1;
461 if (captureMode())
462 t = 0;
463 else if (playlistMode())
464 t = _player->activeAudioTrack();
465 return t;
466 }
467
468 int Bino::subtitleTrack() const
469 {
470 int t = -1;
471 if (captureMode())
472 t = 0;
473 else if (playlistMode())
474 t = _player->activeSubtitleTrack();
475 return t;
476 }
477
478 InputMode Bino::inputMode() const
479 {
480 return _videoSink->inputMode;
481 }
482
483 InputMode Bino::assumeInputMode() const
484 {
485 return _frame.inputMode;
486 }
487
488 bool Bino::assumeStereoInputMode() const
489 {
490 return (assumeInputMode() != Input_Mono);
491 }
492
493 SurroundMode Bino::surroundMode() const
494 {
495 return _videoSink->surroundMode;
496 }
497
498 SurroundMode Bino::assumeSurroundMode() const
499 {
500 return _frame.surroundMode;
501 }
502
503 void Bino::serializeStaticData(QDataStream& ds) const
504 {
505 ds << _screenType << _screen;
506 }
507
508 void Bino::deserializeStaticData(QDataStream& ds)
509 {
510 ds >> _screenType >> _screen;
511 }
512
513 void Bino::serializeDynamicData(QDataStream& ds)
514 {
515 ds << _frameWasSerialized;
516 if (!_frameWasSerialized) {
517 ds << _frame;
518 if (_frame.inputMode == Input_Alternating_LR
519 || _frame.inputMode == Input_Alternating_RL) {
520 ds << _extFrame;
521 }
522 _frameWasSerialized = true;
523 }
524 ds << _swapEyes;
525 }
526
527 void Bino::deserializeDynamicData(QDataStream& ds)
528 {
529 bool noNewFrame;
530 ds >> noNewFrame;
531 if (!noNewFrame) {
532 ds >> _frame;
533 if (_frame.inputMode == Input_Alternating_LR
534 || _frame.inputMode == Input_Alternating_RL) {
535 ds >> _extFrame;
536 }
537 _frameIsNew = true;
538 }
539 ds >> _swapEyes;
540 }
541
542 bool Bino::wantExit() const
543 {
544 return _wantExit;
545 }
546
547 bool Bino::initProcess()
548 {
549 bool haveAnisotropicFiltering = checkTextureAnisotropicFilterAvailability();
550 LOG_DEBUG("Using OpenGL in the %s variant",
551 OpenGLType == OpenGL_Type_WebGL ? "WebGL" : OpenGLType == OpenGL_Type_OpenGLES ? "ES" : "Desktop");
552
553 // Qt-based OpenGL initialization
554 initializeOpenGLFunctions();
555
556 // FBO and PBO
557 glGenFramebuffers(1, &_viewFbo);
558 glGenFramebuffers(1, &_frameFbo);
559 glGenTextures(1, &_depthTex);
560 glBindTexture(GL_TEXTURE_2D, _depthTex);
561 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
562 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
563 glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT24, 1, 1,
564 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, nullptr);
565 glBindFramebuffer(GL_FRAMEBUFFER, _viewFbo);
566 glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, _depthTex, 0);
567 CHECK_GL();
568
569 // Quad geometry
570 const float quadPositions[] = {
571 -1.0f, +1.0f, 0.0f,
572 +1.0f, +1.0f, 0.0f,
573 +1.0f, -1.0f, 0.0f,
574 -1.0f, -1.0f, 0.0f
575 };
576 const float quadTexCoords[] = {
577 0.0f, 1.0f,
578 1.0f, 1.0f,
579 1.0f, 0.0f,
580 0.0f, 0.0f
581 };
582 static const unsigned short quadIndices[] = {
583 0, 3, 1, 1, 3, 2
584 };
585 glGenVertexArrays(1, &_quadVao);
586 glBindVertexArray(_quadVao);
587 GLuint quadPositionBuf;
588 glGenBuffers(1, &quadPositionBuf);
589 glBindBuffer(GL_ARRAY_BUFFER, quadPositionBuf);
590 glBufferData(GL_ARRAY_BUFFER, sizeof(quadPositions), quadPositions, GL_STATIC_DRAW);
591 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, 0);
592 glEnableVertexAttribArray(0);
593 GLuint quadTexCoordBuf;
594 glGenBuffers(1, &quadTexCoordBuf);
595 glBindBuffer(GL_ARRAY_BUFFER, quadTexCoordBuf);
596 glBufferData(GL_ARRAY_BUFFER, sizeof(quadTexCoords), quadTexCoords, GL_STATIC_DRAW);
597 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, 0);
598 glEnableVertexAttribArray(1);
599 GLuint quadIndexBuf;
600 glGenBuffers(1, &quadIndexBuf);
601 glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, quadIndexBuf);
602 glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(quadIndices), quadIndices, GL_STATIC_DRAW);
603 CHECK_GL();
604
605 // Cube geometry
606 const float cubePositions[] = {
607 -10.0f, -10.0f, +10.0f,
608 +10.0f, -10.0f, +10.0f,
609 -10.0f, +10.0f, +10.0f,
610 +10.0f, +10.0f, +10.0f,
611 +10.0f, -10.0f, -10.0f,
612 -10.0f, -10.0f, -10.0f,
613 +10.0f, +10.0f, -10.0f,
614 -10.0f, +10.0f, -10.0f,
615 -10.0f, -10.0f, -10.0f,
616 -10.0f, -10.0f, +10.0f,
617 -10.0f, +10.0f, -10.0f,
618 -10.0f, +10.0f, +10.0f,
619 +10.0f, -10.0f, +10.0f,
620 +10.0f, -10.0f, -10.0f,
621 +10.0f, +10.0f, +10.0f,
622 +10.0f, +10.0f, -10.0f,
623 -10.0f, +10.0f, -10.0f,
624 -10.0f, +10.0f, +10.0f,
625 +10.0f, +10.0f, -10.0f,
626 +10.0f, +10.0f, +10.0f,
627 +10.0f, -10.0f, -10.0f,
628 +10.0f, -10.0f, +10.0f,
629 -10.0f, -10.0f, -10.0f,
630 -10.0f, -10.0f, +10.0f
631 };
632 const float cubeTexCoords[] = {
633 0.0f, 0.0f,
634 1.0f, 0.0f,
635 0.0f, 1.0f,
636 1.0f, 1.0f,
637 0.0f, 0.0f,
638 1.0f, 0.0f,
639 0.0f, 1.0f,
640 1.0f, 1.0f,
641 0.0f, 0.0f,
642 1.0f, 0.0f,
643 0.0f, 1.0f,
644 1.0f, 1.0f,
645 0.0f, 0.0f,
646 1.0f, 0.0f,
647 0.0f, 1.0f,
648 1.0f, 1.0f,
649 0.0f, 0.0f,
650 1.0f, 0.0f,
651 0.0f, 1.0f,
652 1.0f, 1.0f,
653 0.0f, 0.0f,
654 1.0f, 0.0f,
655 0.0f, 1.0f,
656 1.0f, 1.0f
657 };
658 static const unsigned short cubeIndices[] = {
659 0, 1, 2, 1, 3, 2,
660 4, 5, 6, 5, 7, 6,
661 8, 9, 10, 9, 11, 10,
662 12, 13, 14, 13, 15, 14,
663 16, 17, 18, 17, 19, 18,
664 20, 21, 22, 21, 23, 22
665 };
666 glGenVertexArrays(1, &_cubeVao);
667 glBindVertexArray(_cubeVao);
668 GLuint cubePositionBuf;
669 glGenBuffers(1, &cubePositionBuf);
670 glBindBuffer(GL_ARRAY_BUFFER, cubePositionBuf);
671 glBufferData(GL_ARRAY_BUFFER, sizeof(cubePositions), cubePositions, GL_STATIC_DRAW);
672 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, 0);
673 glEnableVertexAttribArray(0);
674 GLuint cubeTexCoordBuf;
675 glGenBuffers(1, &cubeTexCoordBuf);
676 glBindBuffer(GL_ARRAY_BUFFER, cubeTexCoordBuf);
677 glBufferData(GL_ARRAY_BUFFER, sizeof(cubeTexCoords), cubeTexCoords, GL_STATIC_DRAW);
678 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, 0);
679 glEnableVertexAttribArray(1);
680 GLuint cubeIndexBuf;
681 glGenBuffers(1, &cubeIndexBuf);
682 glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, cubeIndexBuf);
683 glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(cubeIndices), cubeIndices, GL_STATIC_DRAW);
684 CHECK_GL();
685
686 // Plane textures
687 glGenTextures(3, _planeTexs);
688 for (int p = 0; p < 3; p++) {
689 glBindTexture(GL_TEXTURE_2D, _planeTexs[p]);
690 unsigned int black = 0;
691 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, &black);
692 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
693 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
694 if (p == 0) {
695 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
696 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
697 } else {
698 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
699 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
700 }
701 }
702 CHECK_GL();
703
704 // Frame textures
705 glGenTextures(1, &_frameTex);
706 glBindTexture(GL_TEXTURE_2D, _frameTex);
707 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
708 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
709 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
710 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
711 if (haveAnisotropicFiltering)
712 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, 4.0f);
713 glGenTextures(1, &_extFrameTex);
714 glBindTexture(GL_TEXTURE_2D, _extFrameTex);
715 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
716 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
717 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
718 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
719 if (haveAnisotropicFiltering)
720 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, 4.0f);
721 CHECK_GL();
722
723 // Subtitle texture
724 glGenTextures(1, &_subtitleTex);
725 glBindTexture(GL_TEXTURE_2D, _subtitleTex);
726 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
727 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
728 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
729 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
730 if (haveAnisotropicFiltering)
731 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, 4.0f);
732 CHECK_GL();
733
734 // Screen geometry
735 glGenVertexArrays(1, &_screenVao);
736 glBindVertexArray(_screenVao);
737 glGenBuffers(1, &_positionBuf);
738 glBindBuffer(GL_ARRAY_BUFFER, _positionBuf);
739 glBufferData(GL_ARRAY_BUFFER, _screen.positions.size() * sizeof(float),
740 _screen.positions.constData(), GL_STATIC_DRAW);
741 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, 0);
742 glEnableVertexAttribArray(0);
743 glGenBuffers(1, &_texcoordBuf);
744 glBindBuffer(GL_ARRAY_BUFFER, _texcoordBuf);
745 glBufferData(GL_ARRAY_BUFFER, _screen.texcoords.size() * sizeof(float),
746 _screen.texcoords.constData(), GL_STATIC_DRAW);
747 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, 0);
748 glEnableVertexAttribArray(1);
749 glGenBuffers(1, &_indexBuf);
750 glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indexBuf);
751 glBufferData(GL_ELEMENT_ARRAY_BUFFER,
752 _screen.indices.length() * sizeof(unsigned int),
753 _screen.indices.constData(), GL_STATIC_DRAW);
754 CHECK_GL();
755
756 return true;
757 }
758
759 void Bino::rebuildColorPrgIfNecessary(int planeFormat, bool colorRangeSmall, int colorSpace, int colorTransfer)
760 {
761 if (_colorPrg.isLinked()
762 && _colorPrgPlaneFormat == planeFormat
763 && _colorPrgColorRangeSmall == colorRangeSmall
764 && _colorPrgColorSpace == colorSpace
765 && _colorPrgColorTransfer == colorTransfer) {
766 return;
767 }
768
769 LOG_DEBUG("rebuilding color conversion program for plane format %d, value range %s, color space %s, color transfer %s",
770 planeFormat, colorRangeSmall ? "small" : "full",
771 colorSpace == VideoFrame::CS_BT601 ? "bt601"
772 : colorSpace == VideoFrame::CS_BT709 ? "bt709"
773 : colorSpace == VideoFrame::CS_AdobeRgb ? "rgb"
774 : "bt2020",
775 colorTransfer == VideoFrame::CT_NOOP ? "none"
776 : colorTransfer == VideoFrame::CT_ST2084 ? "st2084"
777 : "std_b67");
778 QString colorVS = readFile(":src/shader-color.vert.glsl");
779 QString colorFS = readFile(":src/shader-color.frag.glsl");
780 colorFS.replace("$PLANE_FORMAT", QString::number(planeFormat));
781 colorFS.replace("$COLOR_RANGE_SMALL", colorRangeSmall ? "true" : "false");
782 colorFS.replace("$COLOR_SPACE", QString::number(colorSpace));
783 colorFS.replace("$COLOR_TRANSFER", QString::number(colorTransfer));
784 if (OpenGLType != OpenGL_Type_Desktop) {
785 colorVS.prepend("#version 300 es\n");
786 colorFS.prepend("#version 300 es\n"
787 "precision mediump float;\n");
788 } else {
789 colorVS.prepend("#version 330\n");
790 colorFS.prepend("#version 330\n");
791 }
792 _colorPrg.removeAllShaders();
793 _colorPrg.addShaderFromSourceCode(QOpenGLShader::Vertex, colorVS);
794 _colorPrg.addShaderFromSourceCode(QOpenGLShader::Fragment, colorFS);
795 _colorPrg.link();
796 _colorPrgPlaneFormat = planeFormat;
797 _colorPrgColorRangeSmall = colorRangeSmall;
798 _colorPrgColorSpace = colorSpace;
799 _colorPrgColorTransfer = colorTransfer;
800 }
801
802 void Bino::rebuildViewPrgIfNecessary(SurroundMode surroundMode, bool nonLinearOutput)
803 {
804 if (_viewPrg.isLinked()
805 && _viewPrgSurroundMode == surroundMode
806 && _viewPrgNonlinearOutput == nonLinearOutput)
807 return;
808
809 LOG_DEBUG("rebuilding view program for surround mode %s, non linear output %s",
810 surroundModeToString(surroundMode), nonLinearOutput ? "true" : "false");
811 QString viewVS = readFile(":src/shader-view.vert.glsl");
812 QString viewFS = readFile(":src/shader-view.frag.glsl");
813 viewFS.replace("$SURROUND_DEGREES",
814 surroundMode == Surround_360 ? "360"
815 : surroundMode == Surround_180 ? "180"
816 : "0");
817 viewFS.replace("$NONLINEAR_OUTPUT", nonLinearOutput ? "true" : "false");
818 if (OpenGLType != OpenGL_Type_Desktop) {
819 viewVS.prepend("#version 300 es\n");
820 viewFS.prepend("#version 300 es\n"
821 "precision mediump float;\n");
822 } else {
823 viewVS.prepend("#version 330\n");
824 viewFS.prepend("#version 330\n");
825 }
826 _viewPrg.removeAllShaders();
827 _viewPrg.addShaderFromSourceCode(QOpenGLShader::Vertex, viewVS);
828 _viewPrg.addShaderFromSourceCode(QOpenGLShader::Fragment, viewFS);
829 _viewPrg.link();
830 _viewPrgSurroundMode = surroundMode;
831 _viewPrgNonlinearOutput = nonLinearOutput;
832 }
833
834 bool Bino::drawSubtitleToImage(int w, int h, const QString& string)
835 {
836 if (_subtitleImg.width() == w && _subtitleImg.height() == h && _subtitleImgString == string)
837 return false;
838
839 if (_subtitleImg.width() != w || _subtitleImg.height() != h)
840 _subtitleImg = QImage(w, h, QImage::Format_ARGB32_Premultiplied);
841 _subtitleImgString = string;
842
843 QColor bgColor = Qt::black;
844 bgColor.setAlpha(0);
845 _subtitleImg.fill(bgColor);
846 if (string.isEmpty())
847 return true;
848
849 // this tries to reproduce what qvideotexturehelper.cpp does since it is entirely
850 // unclear and undocumented how subtitles are expected to be handled
851
852 QFont font;
853 float fontSize = h * 0.045f;
854 font.setPointSize(fontSize);
855 QTextLayout layout;
856 layout.setText(string);
857 layout.setFont(font);
858 QTextOption option;
859 option.setUseDesignMetrics(true);
860 option.setAlignment(Qt::AlignCenter);
861 layout.setTextOption(option);
862 QFontMetrics metrics(font);
863 float lineWidth = w * 0.9f;
864 float margin = w * 0.05f;
865 float height = 0.0f;
866 float textWidth = 0.0f;
867 layout.beginLayout();
868 for (;;) {
869 QTextLine line = layout.createLine();
870 if (!line.isValid())
871 break;
872 line.setLineWidth(lineWidth);
873 height += metrics.leading();
874 line.setPosition(QPointF(margin, height));
875 height += line.height();
876 textWidth = qMax(textWidth, line.naturalTextWidth());
877 }
878 layout.endLayout();
879 int bottomMargin = h / 20;
880 float y = h - bottomMargin - height;
881 layout.setPosition(QPointF(0.0f, y));
882 textWidth += fontSize / 4.0f;
883 //QRectF bounds = QRectF((w - textWidth) * 0.5f, y, textWidth, height);
884
885 QPainter painter(&_subtitleImg);
886 QTextLayout::FormatRange range;
887 range.start = 0;
888 range.length = layout.text().size();
889 range.format.setForeground(Qt::white);
890 layout.draw(&painter, {}, { range });
891
892 return true;
893 }
894
895 static int alignmentFromBytesPerLine(const void* data, int bpl)
896 {
897 int alignment = 1;
898 if (uint64_t(data) % 8 == 0 && bpl % 8 == 0)
899 alignment = 8;
900 else if (uint64_t(data) % 4 == 0 && bpl % 4 == 0)
901 alignment = 4;
902 else if (uint64_t(data) % 2 == 0 && bpl % 2 == 0)
903 alignment = 2;
904 LOG_FIREHOSE("convertFrameToTexture: alignment is %d (from data %p, bpl %d)", alignment, data, bpl);
905 return alignment;
906 }
907
908 void Bino::convertFrameToTexture(const VideoFrame& frame, unsigned int frameTex)
909 {
910 // 1. Get the frame data into plane textures
911 int w = frame.width;
912 int h = frame.height;
913 int planeFormat; // see shader-color.frag.glsl
914 int planeCount;
915 // reset swizzling for plane0; might be changed below depending in the format
916 glBindTexture(GL_TEXTURE_2D, _planeTexs[0]);
917 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_RED);
918 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
919 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_BLUE);
920 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_ALPHA);
921 if (frame.storage == VideoFrame::Storage_Image) {
922 LOG_FIREHOSE("convertFrameToTexture: format is image");
923 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(frame.image.constBits(), frame.image.bytesPerLine()));
924 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.image.bytesPerLine() / 4);
925 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, frame.image.constBits());
926 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
927 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
928 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
929 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_ALPHA);
930 planeFormat = 1;
931 planeCount = 1;
932 } else {
933 std::array<const void*, 3> planeData;
934 if (frame.storage == VideoFrame::Storage_Mapped) {
935 planeData = { frame.mappedBits[0], frame.mappedBits[1], frame.mappedBits[2] };
936 } else {
937 planeData = { frame.bits[0].data(), frame.bits[1].data(), frame.bits[2].data() };
938 }
939 if (frame.pixelFormat == QVideoFrameFormat::Format_ARGB8888
940 || frame.pixelFormat == QVideoFrameFormat::Format_ARGB8888_Premultiplied
941 || frame.pixelFormat == QVideoFrameFormat::Format_XRGB8888) {
942 LOG_FIREHOSE("convertFrameToTexture: format argb8888");
943 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
944 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 4);
945 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, planeData[0]);
946 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_ALPHA);
947 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_RED);
948 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_GREEN);
949 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_BLUE);
950 planeFormat = 1;
951 planeCount = 1;
952 } else if (frame.pixelFormat == QVideoFrameFormat::Format_BGRA8888
953 || frame.pixelFormat == QVideoFrameFormat::Format_BGRA8888_Premultiplied
954 || frame.pixelFormat == QVideoFrameFormat::Format_BGRX8888) {
955 LOG_FIREHOSE("convertFrameToTexture: format bgra8888");
956 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
957 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 4);
958 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, planeData[0]);
959 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
960 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
961 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
962 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_ALPHA);
963 planeFormat = 1;
964 planeCount = 1;
965 } else if (frame.pixelFormat == QVideoFrameFormat::Format_ABGR8888
966 || frame.pixelFormat == QVideoFrameFormat::Format_XBGR8888) {
967 LOG_FIREHOSE("convertFrameToTexture: format abgr8888");
968 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
969 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 4);
970 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, planeData[0]);
971 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_ALPHA);
972 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_BLUE);
973 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_GREEN);
974 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_RED);
975 planeFormat = 1;
976 planeCount = 1;
977 } else if (frame.pixelFormat == QVideoFrameFormat::Format_RGBA8888
978 || frame.pixelFormat == QVideoFrameFormat::Format_RGBX8888) {
979 LOG_FIREHOSE("convertFrameToTexture: format rgba8888");
980 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
981 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 4);
982 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, planeData[0]);
983 planeFormat = 1;
984 planeCount = 1;
985 } else if (frame.pixelFormat == QVideoFrameFormat::Format_YUV420P) {
986 LOG_FIREHOSE("convertFrameToTexture: format yuv420p");
987 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
988 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0));
989 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[0]);
990 glBindTexture(GL_TEXTURE_2D, _planeTexs[1]);
991 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[1], frame.qframe.bytesPerLine(1)));
992 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(1));
993 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h / 2, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[1]);
994 glBindTexture(GL_TEXTURE_2D, _planeTexs[2]);
995 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[2], frame.qframe.bytesPerLine(2)));
996 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(2));
997 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h / 2, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[2]);
998 planeFormat = 2;
999 planeCount = 3;
1000 } else if (frame.pixelFormat == QVideoFrameFormat::Format_YUV422P) {
1001 LOG_FIREHOSE("convertFrameToTexture: format yuv422p");
1002 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1003 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0));
1004 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[0]);
1005 glBindTexture(GL_TEXTURE_2D, _planeTexs[1]);
1006 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[1], frame.qframe.bytesPerLine(1)));
1007 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(1));
1008 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[1]);
1009 glBindTexture(GL_TEXTURE_2D, _planeTexs[2]);
1010 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[2], frame.qframe.bytesPerLine(2)));
1011 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(2));
1012 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[2]);
1013 planeFormat = 2;
1014 planeCount = 3;
1015 } else if (frame.pixelFormat == QVideoFrameFormat::Format_YV12) {
1016 LOG_FIREHOSE("convertFrameToTexture: format yv12");
1017 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1018 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0));
1019 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[0]);
1020 glBindTexture(GL_TEXTURE_2D, _planeTexs[1]);
1021 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[1], frame.qframe.bytesPerLine(1)));
1022 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(1));
1023 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h / 2, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[1]);
1024 glBindTexture(GL_TEXTURE_2D, _planeTexs[2]);
1025 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[2], frame.qframe.bytesPerLine(2)));
1026 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(2));
1027 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w / 2, h / 2, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[2]);
1028 planeFormat = 3;
1029 planeCount = 3;
1030 } else if (frame.pixelFormat == QVideoFrameFormat::Format_NV12) {
1031 LOG_FIREHOSE("convertFrameToTexture: format nv12");
1032 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1033 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0));
1034 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[0]);
1035 glBindTexture(GL_TEXTURE_2D, _planeTexs[1]);
1036 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[1], frame.qframe.bytesPerLine(1)));
1037 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(1) / 2);
1038 glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, w / 2, h / 2, 0, GL_RG, GL_UNSIGNED_BYTE, planeData[1]);
1039 planeFormat = 4;
1040 planeCount = 2;
1041 } else if (frame.pixelFormat == QVideoFrameFormat::Format_P010
1042 || frame.pixelFormat == QVideoFrameFormat::Format_P016) {
1043 LOG_FIREHOSE("convertFrameToTexture: format p010/p016");
1044 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1045 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 2);
1046 glTexImage2D(GL_TEXTURE_2D, 0, GL_R16, w, h, 0, GL_RED, GL_UNSIGNED_SHORT, planeData[0]);
1047 glBindTexture(GL_TEXTURE_2D, _planeTexs[1]);
1048 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[1], frame.qframe.bytesPerLine(1)));
1049 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(1) / 4);
1050 glTexImage2D(GL_TEXTURE_2D, 0, GL_RG16, w / 2, h / 2, 0, GL_RG, GL_UNSIGNED_SHORT, planeData[1]);
1051 planeFormat = 4;
1052 planeCount = 2;
1053 } else if (frame.pixelFormat == QVideoFrameFormat::Format_Y8) {
1054 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1055 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0));
1056 LOG_FIREHOSE("convertFrameToTexture: format y8");
1057 glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, planeData[0]);
1058 planeFormat = 5;
1059 planeCount = 1;
1060 } else if (frame.pixelFormat == QVideoFrameFormat::Format_Y16) {
1061 glPixelStorei(GL_UNPACK_ALIGNMENT, alignmentFromBytesPerLine(planeData[0], frame.qframe.bytesPerLine(0)));
1062 glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.qframe.bytesPerLine(0) / 2);
1063 LOG_FIREHOSE("convertFrameToTexture: format y16");
1064 glTexImage2D(GL_TEXTURE_2D, 0, GL_R16, w, h, 0, GL_RED, GL_UNSIGNED_SHORT, planeData[0]);
1065 planeFormat = 5;
1066 planeCount = 1;
1067 } else {
1068 LOG_FATAL("Unhandled pixel format");
1069 std::exit(1);
1070 }
1071 }
1072 glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
1073 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
1074 // 2. Convert plane textures into linear RGB in the frame texture
1075 glBindTexture(GL_TEXTURE_2D, frameTex);
1076 if (OpenGLType == OpenGL_Type_WebGL)
1077 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_BGRA, GL_UNSIGNED_SHORT, nullptr);
1078 else if (OpenGLType == OpenGL_Type_OpenGLES)
1079 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, w, h, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr);
1080 else
1081 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA16, w, h, 0, GL_BGRA, GL_UNSIGNED_SHORT, nullptr);
1082 glBindFramebuffer(GL_FRAMEBUFFER, _frameFbo);
1083 glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, frameTex, 0);
1084 glViewport(0, 0, w, h);
1085 glDisable(GL_DEPTH_TEST);
1086 rebuildColorPrgIfNecessary(planeFormat, frame.colorRangeSmall, frame.colorSpace, frame.colorTransfer);
1087 glUseProgram(_colorPrg.programId());
1088 _colorPrg.setUniformValue("masteringWhite", frame.masteringWhite);
1089 for (int p = 0; p < planeCount; p++) {
1090 _colorPrg.setUniformValue(qPrintable(QString("plane") + QString::number(p)), p);
1091 glActiveTexture(GL_TEXTURE0 + p);
1092 glBindTexture(GL_TEXTURE_2D, _planeTexs[p]);
1093 }
1094 glBindVertexArray(_quadVao);
1095 glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1096 glBindTexture(GL_TEXTURE_2D, frameTex);
1097 glGenerateMipmap(GL_TEXTURE_2D);
1098 }
1099
1100 void Bino::preRenderProcess(int screenWidth, int screenHeight,
1101 int* viewCountPtr, int* viewWidthPtr, int* viewHeightPtr, float* frameDisplayAspectRatioPtr, bool* surroundPtr)
1102 {
1103 Q_ASSERT(_frame.inputMode != Input_Unknown);
1104
1105 int viewCount = 2;
1106 int viewWidth = _frame.width;
1107 int viewHeight = _frame.height;
1108 float frameDisplayAspectRatio = _frame.aspectRatio;
1109 switch (_frame.inputMode) {
1110 case Input_Unknown: // cannot happen, update() sets a known mode
1111 case Input_Mono:
1112 viewCount = 1;
1113 break;
1114 case Input_Top_Bottom:
1115 case Input_Bottom_Top:
1116 frameDisplayAspectRatio *= 2.0f;
1117 [[fallthrough]];
1118 case Input_Top_Bottom_Half:
1119 case Input_Bottom_Top_Half:
1120 viewHeight /= 2;
1121 break;
1122 case Input_Left_Right:
1123 case Input_Right_Left:
1124 frameDisplayAspectRatio /= 2.0f;
1125 [[fallthrough]];
1126 case Input_Left_Right_Half:
1127 case Input_Right_Left_Half:
1128 viewWidth /= 2;
1129 break;
1130 case Input_Alternating_LR:
1131 case Input_Alternating_RL:
1132 break;
1133 }
1134 switch (_frame.surroundMode) {
1135 case Surround_Unknown: // cannot happen, update() sets a known mode
1136 case Surround_Off:
1137 break;
1138 case Surround_180:
1139 frameDisplayAspectRatio *= 2.0f;
1140 break;
1141 case Surround_360:
1142 break;
1143 }
1144 if (subtitleTrack() >= 0 && (screenWidth > viewWidth || screenHeight > viewHeight)) {
1145 if (screenWidth / viewWidth > screenHeight / viewHeight) {
1146 viewWidth = screenWidth;
1147 viewHeight = viewWidth / frameDisplayAspectRatio;
1148 } else {
1149 viewHeight = screenHeight;
1150 viewWidth = viewHeight * frameDisplayAspectRatio;
1151 }
1152 }
1153
1154 if (viewCountPtr)
1155 *viewCountPtr = viewCount;
1156 if (viewWidthPtr)
1157 *viewWidthPtr = viewWidth;
1158 if (viewHeightPtr)
1159 *viewHeightPtr = viewHeight;
1160 if (frameDisplayAspectRatioPtr)
1161 *frameDisplayAspectRatioPtr = frameDisplayAspectRatio;
1162 if (surroundPtr)
1163 *surroundPtr = (_frame.surroundMode != Surround_Off);
1164
1165 /* We need to get new frame data into a texture that is suitable for
1166 * rendering the screen: _frameTex. */
1167
1168 if (_frameIsNew) {
1169 // Convert _frame into _frameTex and, if needed, _extFrame into _extFrameTex.
1170 convertFrameToTexture(_frame, _frameTex);
1171 if (_frame.inputMode == Input_Alternating_LR
1172 || _frame.inputMode == Input_Alternating_RL) {
1173 // the user might have switched to this mode without the extFrame
1174 // being available, in that case fall back to the standard frame
1175 if (_extFrame.width != _frame.width || _extFrame.height != _frame.height)
1176 convertFrameToTexture(_frame, _extFrameTex);
1177 else
1178 convertFrameToTexture(_extFrame, _extFrameTex);
1179 }
1180 // Render the subtitle into the subtitle texture
1181 if (drawSubtitleToImage(viewWidth, viewHeight, _frame.subtitle)) {
1182 glBindTexture(GL_TEXTURE_2D, _subtitleTex);
1183 glTexImage2D(GL_TEXTURE_2D, 0, GL_SRGB8_ALPHA8,
1184 _subtitleImg.width(), _subtitleImg.height(), 0, GL_RGBA,
1185 GL_UNSIGNED_BYTE, _subtitleImg.bits());
1186 glGenerateMipmap(GL_TEXTURE_2D);
1187 }
1188 // Done.
1189 _frameIsNew = false;
1190 }
1191 if (_frame.inputMode != _lastFrameInputMode
1192 || _frame.surroundMode != _lastFrameSurroundMode) {
1193 emit stateChanged();
1194 }
1195 _lastFrameInputMode = _frame.inputMode;
1196 _lastFrameSurroundMode = _frame.surroundMode;
1197 }
1198
1199 void Bino::render(
1200 const QVector3D& unitedScreenBottomLeft, const QVector3D& unitedScreenBottomRight, const QVector3D& unitedScreenTopLeft,
1201 const QVector3D& intersectedScreenBottomLeft, const QVector3D& intersectedScreenBottomRight, const QVector3D& intersectedScreenTopLeft,
1202 const QMatrix4x4& projectionMatrix,
1203 const QMatrix4x4& orientationMatrix,
1204 const QMatrix4x4& viewMatrix,
1205 int view, // 0 = left, 1 = right
1206 int texWidth, int texHeight, unsigned int texture)
1207 {
1208 // Update screen
1209 switch (_screenType) {
1210 case ScreenUnited:
1211 _screen = Screen(unitedScreenBottomLeft, unitedScreenBottomRight, unitedScreenTopLeft);
1212 break;
1213 case ScreenIntersected:
1214 _screen = Screen(intersectedScreenBottomLeft, intersectedScreenBottomRight, intersectedScreenTopLeft);
1215 break;
1216 case ScreenGeometry:
1217 // do nothing
1218 break;
1219 }
1220 // Set up framebuffer object to render into
1221 glBindTexture(GL_TEXTURE_2D, _depthTex);
1222 glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT24, texWidth, texHeight,
1223 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, nullptr);
1224 glEnable(GL_DEPTH_TEST);
1225 glBindFramebuffer(GL_FRAMEBUFFER, _viewFbo);
1226 glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0);
1227 // Set up view
1228 glViewport(0, 0, texWidth, texHeight);
1229 glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1230 // Set up input mode
1231 unsigned int frameTex = _frameTex;
1232 float frameAspectRatio = _frame.aspectRatio;
1233 float viewOffsetX = 0.0f;
1234 float viewFactorX = 1.0f;
1235 float viewOffsetY = 0.0f;
1236 float viewFactorY = 1.0f;
1237 if (_swapEyes)
1238 view = (view == 0 ? 1 : 0);
1239 switch (_frame.inputMode) {
1240 case Input_Unknown: // cannot happen, update() sets a known mode
1241 case Input_Mono:
1242 // nothing to do
1243 break;
1244 case Input_Top_Bottom:
1245 viewFactorY = 0.5f;
1246 viewOffsetY = (view == 1 ? 0.5f : 0.0f);
1247 frameAspectRatio *= 2.0f;
1248 break;
1249 case Input_Top_Bottom_Half:
1250 viewFactorY = 0.5f;
1251 viewOffsetY = (view == 1 ? 0.5f : 0.0f);
1252 break;
1253 case Input_Bottom_Top:
1254 viewFactorY = 0.5f;
1255 viewOffsetY = (view != 1 ? 0.5f : 0.0f);
1256 frameAspectRatio *= 2.0f;
1257 break;
1258 case Input_Bottom_Top_Half:
1259 viewFactorY = 0.5f;
1260 viewOffsetY = (view != 1 ? 0.5f : 0.0f);
1261 break;
1262 case Input_Left_Right:
1263 viewFactorX = 0.5f;
1264 viewOffsetX = (view == 1 ? 0.5f : 0.0f);
1265 frameAspectRatio /= 2.0f;
1266 break;
1267 case Input_Left_Right_Half:
1268 viewFactorX = 0.5f;
1269 viewOffsetX = (view == 1 ? 0.5f : 0.0f);
1270 break;
1271 case Input_Right_Left:
1272 viewFactorX = 0.5f;
1273 viewOffsetX = (view != 1 ? 0.5f : 0.0f);
1274 frameAspectRatio /= 2.0f;
1275 break;
1276 case Input_Right_Left_Half:
1277 viewFactorX = 0.5f;
1278 viewOffsetX = (view != 1 ? 0.5f : 0.0f);
1279 break;
1280 case Input_Alternating_LR:
1281 if (view == 1)
1282 frameTex = _extFrameTex;
1283 break;
1284 case Input_Alternating_RL:
1285 if (view == 0)
1286 frameTex = _extFrameTex;
1287 break;
1288 }
1289 LOG_FIREHOSE("Rendering view %d from %s frame texture fx=%g ox=%g fy=%g oy=%g",
1290 view, frameTex == _frameTex ? "standard" : "extended", viewFactorX, viewOffsetX, viewFactorY, viewOffsetY);
1291 // Determine if we are producing the final rendering result here (which is the
1292 // case for VR mode) or if we are just rendering to intermediate textures (which
1293 // is the case for GUI mode). In GUI mode, the screen aspect ratio is unknown.
1294 bool finalRenderingStep = (_screen.aspectRatio > 0.0f);
1295 // Set up correct aspect ratio on screen
1296 float relWidth = 1.0f;
1297 float relHeight = 1.0f;
1298 if (finalRenderingStep) {
1299 if (_screen.aspectRatio < frameAspectRatio)
1300 relHeight = _screen.aspectRatio / frameAspectRatio;
1301 else
1302 relWidth = frameAspectRatio / _screen.aspectRatio;
1303 }
1304 // Set up shader program
1305 rebuildViewPrgIfNecessary(_frame.surroundMode, finalRenderingStep);
1306 glUseProgram(_viewPrg.programId());
1307 QMatrix4x4 projectionModelViewMatrix = projectionMatrix;
1308 if (_frame.surroundMode == Surround_Off)
1309 projectionModelViewMatrix = projectionModelViewMatrix * viewMatrix;
1310 _viewPrg.setUniformValue("projectionModelViewMatrix", projectionModelViewMatrix);
1311 _viewPrg.setUniformValue("orientationMatrix", orientationMatrix);
1312 _viewPrg.setUniformValue("frameTex", 0);
1313 _viewPrg.setUniformValue("subtitleTex", 1);
1314 _viewPrg.setUniformValue("view_offset_x", viewOffsetX);
1315 _viewPrg.setUniformValue("view_factor_x", viewFactorX);
1316 _viewPrg.setUniformValue("view_offset_y", viewOffsetY);
1317 _viewPrg.setUniformValue("view_factor_y", viewFactorY);
1318 _viewPrg.setUniformValue("relative_width", relWidth);
1319 _viewPrg.setUniformValue("relative_height", relHeight);
1320 // Render scene
1321 glActiveTexture(GL_TEXTURE1);
1322 glBindTexture(GL_TEXTURE_2D, _subtitleTex);
1323 glActiveTexture(GL_TEXTURE0);
1324 glBindTexture(GL_TEXTURE_2D, frameTex);
1325 if (_frame.surroundMode != Surround_Off) {
1326 // Set up filtering to work correctly at the horizontal wraparound:
1327 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
1328 if (_frame.surroundMode == Surround_360)
1329 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
1330 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
1331 // Render
1332 glBindVertexArray(_cubeVao);
1333 glDrawElements(GL_TRIANGLES, 36, GL_UNSIGNED_SHORT, 0);
1334 // Reset filtering parameters to their defaults
1335 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
1336 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
1337 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
1338 } else {
1339 glBindVertexArray(_screenVao);
1340 if (_screenType == ScreenUnited || _screenType == ScreenIntersected) {
1341 // the geometry might have changed; re-upload it
1342 glBindBuffer(GL_ARRAY_BUFFER, _positionBuf);
1343 glBufferData(GL_ARRAY_BUFFER, _screen.positions.size() * sizeof(float),
1344 _screen.positions.constData(), GL_STATIC_DRAW);
1345 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, 0);
1346 glBindBuffer(GL_ARRAY_BUFFER, _texcoordBuf);
1347 glBufferData(GL_ARRAY_BUFFER, _screen.texcoords.size() * sizeof(float),
1348 _screen.texcoords.constData(), GL_STATIC_DRAW);
1349 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, 0);
1350 glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indexBuf);
1351 glBufferData(GL_ELEMENT_ARRAY_BUFFER,
1352 _screen.indices.length() * sizeof(unsigned int),
1353 _screen.indices.constData(), GL_STATIC_DRAW);
1354 }
1355 glDrawElements(GL_TRIANGLES, _screen.indices.size(), GL_UNSIGNED_INT, 0);
1356 }
1357 }
1358
1359 void Bino::keyPressEvent(QKeyEvent* event)
1360 {
1361 if (event->matches(QKeySequence::Quit)
1362 || event->matches(QKeySequence::Cancel)
1363 || event->key() == Qt::Key_Escape
1364 || event->key() == Qt::Key_MediaStop) {
1365 quit();
1366 } else if (event->key() == Qt::Key_MediaTogglePlayPause
1367 || event->key() == Qt::Key_Space) {
1368 togglePause();
1369 } else if (event->key() == Qt::Key_MediaPause) {
1370 pause();
1371 } else if (event->key() == Qt::Key_MediaPlay) {
1372 play();
1373 } else if (event->key() == Qt::Key_VolumeMute || event->key() == Qt::Key_M) {
1374 toggleMute();
1375 } else if (event->key() == Qt::Key_VolumeDown) {
1376 changeVolume(-0.05f);
1377 } else if (event->key() == Qt::Key_VolumeUp) {
1378 changeVolume(+0.05f);
1379 } else if (event->key() == Qt::Key_Period) {
1380 seek(+1000);
1381 } else if (event->key() == Qt::Key_Comma) {
1382 seek(-1000);
1383 } else if (event->key() == Qt::Key_Right) {
1384 seek(+10000);
1385 } else if (event->key() == Qt::Key_Left) {
1386 seek(-10000);
1387 } else if (event->key() == Qt::Key_Right) {
1388 seek(+10000);
1389 } else if (event->key() == Qt::Key_Down) {
1390 seek(-60000);
1391 } else if (event->key() == Qt::Key_Up) {
1392 seek(+60000);
1393 } else if (event->key() == Qt::Key_PageDown) {
1394 seek(-600000);
1395 } else if (event->key() == Qt::Key_PageUp) {
1396 seek(+600000);
1397 } else if (event->key() == Qt::Key_N) {
1398 Playlist::instance()->next();
1399 } else if (event->key() == Qt::Key_P) {
1400 Playlist::instance()->prev();
1401 } else if (event->matches(QKeySequence::FullScreen) || event->key() == Qt::Key_F || event->key() == Qt::Key_F11) {
1402 emit toggleFullscreen();
1403 } else if (event->key() == Qt::Key_E || event->key() == Qt::Key_F7) {
1404 toggleSwapEyes();
1405 } else {
1406 LOG_DEBUG("Unhandled key event: key=%d text='%s'", event->key(), qPrintable(event->text()));
1407 event->ignore();
1408 }
1409 }