OpenShot Library | libopenshot  0.2.7
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #include "FFmpegReader.h"
35 #include "Exceptions.h"
36 #include "Timeline.h"
37 
38 #include <thread> // for std::this_thread::sleep_for
39 #include <chrono> // for std::chrono::milliseconds
40 
41 #define ENABLE_VAAPI 0
42 
43 #if USE_HW_ACCEL
44 #define MAX_SUPPORTED_WIDTH 1950
45 #define MAX_SUPPORTED_HEIGHT 1100
46 
47 #if ENABLE_VAAPI
48 #include "libavutil/hwcontext_vaapi.h"
49 
50 typedef struct VAAPIDecodeContext {
51  VAProfile va_profile;
52  VAEntrypoint va_entrypoint;
53  VAConfigID va_config;
54  VAContextID va_context;
55 
56 #if FF_API_STRUCT_VAAPI_CONTEXT
57  // FF_DISABLE_DEPRECATION_WARNINGS
58  int have_old_context;
59  struct vaapi_context *old_context;
60  AVBufferRef *device_ref;
61  // FF_ENABLE_DEPRECATION_WARNINGS
62 #endif
63 
64  AVHWDeviceContext *device;
65  AVVAAPIDeviceContext *hwctx;
66 
67  AVHWFramesContext *frames;
68  AVVAAPIFramesContext *hwfc;
69 
70  enum AVPixelFormat surface_format;
71  int surface_count;
72  } VAAPIDecodeContext;
73 #endif // ENABLE_VAAPI
74 #endif // USE_HW_ACCEL
75 
76 
77 using namespace openshot;
78 
79 int hw_de_on = 0;
80 #if USE_HW_ACCEL
81  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
82  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
83 #endif
84 
85 FFmpegReader::FFmpegReader(const std::string& path, bool inspect_reader)
86  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
87  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
88  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
89  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
90  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
91  packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS) {
92 
93  // Initialize FFMpeg, and register all formats and codecs
96 
97  // Init cache
98  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
99  missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
101 
102  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
103  if (inspect_reader) {
104  Open();
105  Close();
106  }
107 }
108 
110  if (is_open)
111  // Auto close reader if not already done
112  Close();
113 }
114 
115 // This struct holds the associated video frame and starting sample # for an audio packet.
116 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
117  // Is frame even close to this one?
118  if (abs(location.frame - frame) >= 2)
119  // This is too far away to be considered
120  return false;
121 
122  // Note that samples_per_frame can vary slightly frame to frame when the
123  // audio sampling rate is not an integer multiple of the video fps.
124  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
125  if (abs(diff) <= amount)
126  // close
127  return true;
128 
129  // not close
130  return false;
131 }
132 
133 #if USE_HW_ACCEL
134 
135 // Get hardware pix format
136 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
137 {
138  const enum AVPixelFormat *p;
139 
140  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
141  switch (*p) {
142 #if defined(__linux__)
143  // Linux pix formats
144  case AV_PIX_FMT_VAAPI:
145  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
146  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
147  return *p;
148  break;
149  case AV_PIX_FMT_VDPAU:
150  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
151  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
152  return *p;
153  break;
154 #endif
155 #if defined(_WIN32)
156  // Windows pix formats
157  case AV_PIX_FMT_DXVA2_VLD:
158  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
159  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
160  return *p;
161  break;
162  case AV_PIX_FMT_D3D11:
163  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
164  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
165  return *p;
166  break;
167 #endif
168 #if defined(__APPLE__)
169  // Apple pix formats
170  case AV_PIX_FMT_VIDEOTOOLBOX:
171  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
172  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
173  return *p;
174  break;
175 #endif
176  // Cross-platform pix formats
177  case AV_PIX_FMT_CUDA:
178  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
179  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
180  return *p;
181  break;
182  case AV_PIX_FMT_QSV:
183  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
184  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
185  return *p;
186  break;
187  default:
188  // This is only here to silence unused-enum warnings
189  break;
190  }
191  }
192  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
193  return AV_PIX_FMT_NONE;
194 }
195 
196 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
197 {
198  int ret;
199  switch (codecid) {
200  case AV_CODEC_ID_H264:
201  case AV_CODEC_ID_MPEG2VIDEO:
202  case AV_CODEC_ID_VC1:
203  case AV_CODEC_ID_WMV1:
204  case AV_CODEC_ID_WMV2:
205  case AV_CODEC_ID_WMV3:
206  ret = 1;
207  break;
208  default :
209  ret = 0;
210  break;
211  }
212  return ret;
213 }
214 #endif // USE_HW_ACCEL
215 
217  // Open reader if not already open
218  if (!is_open) {
219  // Initialize format context
220  pFormatCtx = NULL;
221  {
223  }
224 
225  // Open video file
226  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
227  throw InvalidFile("File could not be opened.", path);
228 
229  // Retrieve stream information
230  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
231  throw NoStreamsFound("No streams found in file.", path);
232 
233  videoStream = -1;
234  audioStream = -1;
235  // Loop through each stream, and identify the video and audio stream index
236  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
237  // Is this a video stream?
238  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
239  videoStream = i;
240  }
241  // Is this an audio stream?
242  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
243  audioStream = i;
244  }
245  }
246  if (videoStream == -1 && audioStream == -1)
247  throw NoStreamsFound("No video or audio streams found in this file.", path);
248 
249  // Is there a video stream?
250  if (videoStream != -1) {
251  // Set the stream index
252  info.video_stream_index = videoStream;
253 
254  // Set the codec and codec context pointers
255  pStream = pFormatCtx->streams[videoStream];
256 
257  // Find the codec ID from stream
258  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
259 
260  // Get codec and codec context from stream
261  AVCodec *pCodec = avcodec_find_decoder(codecId);
262  AVDictionary *opts = NULL;
263  int retry_decode_open = 2;
264  // If hw accel is selected but hardware cannot handle repeat with software decoding
265  do {
266  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
267 #if USE_HW_ACCEL
268  if (hw_de_on && (retry_decode_open==2)) {
269  // Up to here no decision is made if hardware or software decode
270  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
271  }
272 #endif
273  retry_decode_open = 0;
274 
275  // Set number of threads equal to number of processors (not to exceed 16)
276  pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
277 
278  if (pCodec == NULL) {
279  throw InvalidCodec("A valid video codec could not be found for this file.", path);
280  }
281 
282  // Init options
283  av_dict_set(&opts, "strict", "experimental", 0);
284 #if USE_HW_ACCEL
285  if (hw_de_on && hw_de_supported) {
286  // Open Hardware Acceleration
287  int i_decoder_hw = 0;
288  char adapter[256];
289  char *adapter_ptr = NULL;
290  int adapter_num;
292  fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
293 
294  // Set hardware pix format (callback)
295  pCodecCtx->get_format = get_hw_dec_format;
296 
297  if (adapter_num < 3 && adapter_num >=0) {
298 #if defined(__linux__)
299  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
300  adapter_ptr = adapter;
302  switch (i_decoder_hw) {
303  case 1:
304  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
305  break;
306  case 2:
307  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
308  break;
309  case 6:
310  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
311  break;
312  case 7:
313  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
314  break;
315  default:
316  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
317  break;
318  }
319 
320 #elif defined(_WIN32)
321  adapter_ptr = NULL;
323  switch (i_decoder_hw) {
324  case 2:
325  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
326  break;
327  case 3:
328  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
329  break;
330  case 4:
331  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
332  break;
333  case 7:
334  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
335  break;
336  default:
337  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
338  break;
339  }
340 #elif defined(__APPLE__)
341  adapter_ptr = NULL;
343  switch (i_decoder_hw) {
344  case 5:
345  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
346  break;
347  case 7:
348  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
349  break;
350  default:
351  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
352  break;
353  }
354 #endif
355 
356  } else {
357  adapter_ptr = NULL; // Just to be sure
358  }
359 
360  // Check if it is there and writable
361 #if defined(__linux__)
362  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
363 #elif defined(_WIN32)
364  if( adapter_ptr != NULL ) {
365 #elif defined(__APPLE__)
366  if( adapter_ptr != NULL ) {
367 #endif
368  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
369  }
370  else {
371  adapter_ptr = NULL; // use default
372  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
373  }
374 
375  hw_device_ctx = NULL;
376  // Here the first hardware initialisations are made
377  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
378  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
379  throw InvalidCodec("Hardware device reference create failed.", path);
380  }
381 
382  /*
383  av_buffer_unref(&ist->hw_frames_ctx);
384  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
385  if (!ist->hw_frames_ctx) {
386  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
387  return AVERROR(ENOMEM);
388  }
389 
390  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
391 
392  frames_ctx->format = AV_PIX_FMT_CUDA;
393  frames_ctx->sw_format = avctx->sw_pix_fmt;
394  frames_ctx->width = avctx->width;
395  frames_ctx->height = avctx->height;
396 
397  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
398  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
399 
400 
401  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
402  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
403  if (ret < 0) {
404  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
405  return ret;
406  }
407  */
408  }
409  else {
410  throw InvalidCodec("Hardware device create failed.", path);
411  }
412  }
413 #endif // USE_HW_ACCEL
414 
415  // Disable per-frame threading for album arts
416  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
417  // but there's only one frame in this case.
418  if (HasAlbumArt())
419  {
420  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
421  }
422 
423  // Open video codec
424  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
425  throw InvalidCodec("A video codec was found, but could not be opened.", path);
426 
427 #if USE_HW_ACCEL
428  if (hw_de_on && hw_de_supported) {
429  AVHWFramesConstraints *constraints = NULL;
430  void *hwconfig = NULL;
431  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
432 
433 // TODO: needs va_config!
434 #if ENABLE_VAAPI
435  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
436  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
437 #endif // ENABLE_VAAPI
438  if (constraints) {
439  if (pCodecCtx->coded_width < constraints->min_width ||
440  pCodecCtx->coded_height < constraints->min_height ||
441  pCodecCtx->coded_width > constraints->max_width ||
442  pCodecCtx->coded_height > constraints->max_height) {
443  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
444  hw_de_supported = 0;
445  retry_decode_open = 1;
446  AV_FREE_CONTEXT(pCodecCtx);
447  if (hw_device_ctx) {
448  av_buffer_unref(&hw_device_ctx);
449  hw_device_ctx = NULL;
450  }
451  }
452  else {
453  // All is just peachy
454  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
455  retry_decode_open = 0;
456  }
457  av_hwframe_constraints_free(&constraints);
458  if (hwconfig) {
459  av_freep(&hwconfig);
460  }
461  }
462  else {
463  int max_h, max_w;
464  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
466  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
468  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
469  //cerr << "Constraints could not be found using default limit\n";
470  if (pCodecCtx->coded_width < 0 ||
471  pCodecCtx->coded_height < 0 ||
472  pCodecCtx->coded_width > max_w ||
473  pCodecCtx->coded_height > max_h ) {
474  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
475  hw_de_supported = 0;
476  retry_decode_open = 1;
477  AV_FREE_CONTEXT(pCodecCtx);
478  if (hw_device_ctx) {
479  av_buffer_unref(&hw_device_ctx);
480  hw_device_ctx = NULL;
481  }
482  }
483  else {
484  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
485  retry_decode_open = 0;
486  }
487  }
488  } // if hw_de_on && hw_de_supported
489  else {
490  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
491  }
492 #else
493  retry_decode_open = 0;
494 #endif // USE_HW_ACCEL
495  } while (retry_decode_open); // retry_decode_open
496  // Free options
497  av_dict_free(&opts);
498 
499  // Update the File Info struct with video details (if a video stream is found)
500  UpdateVideoInfo();
501  }
502 
503  // Is there an audio stream?
504  if (audioStream != -1) {
505  // Set the stream index
506  info.audio_stream_index = audioStream;
507 
508  // Get a pointer to the codec context for the audio stream
509  aStream = pFormatCtx->streams[audioStream];
510 
511  // Find the codec ID from stream
512  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
513 
514  // Get codec and codec context from stream
515  AVCodec *aCodec = avcodec_find_decoder(codecId);
516  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
517 
518  // Set number of threads equal to number of processors (not to exceed 16)
519  aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
520 
521  if (aCodec == NULL) {
522  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
523  }
524 
525  // Init options
526  AVDictionary *opts = NULL;
527  av_dict_set(&opts, "strict", "experimental", 0);
528 
529  // Open audio codec
530  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
531  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
532 
533  // Free options
534  av_dict_free(&opts);
535 
536  // Update the File Info struct with audio details (if an audio stream is found)
537  UpdateAudioInfo();
538  }
539 
540  // Add format metadata (if any)
541  AVDictionaryEntry *tag = NULL;
542  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
543  QString str_key = tag->key;
544  QString str_value = tag->value;
545  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
546  }
547 
548  // Init previous audio location to zero
549  previous_packet_location.frame = -1;
550  previous_packet_location.sample_start = 0;
551 
552  // Adjust cache size based on size of frame and audio
553  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
554  missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
556 
557  // Mark as "open"
558  is_open = true;
559  }
560 }
561 
563  // Close all objects, if reader is 'open'
564  if (is_open) {
565  // Mark as "closed"
566  is_open = false;
567 
568  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close");
569 
570  if (packet) {
571  // Remove previous packet before getting next one
572  RemoveAVPacket(packet);
573  packet = NULL;
574  }
575 
576  // Close the codec
577  if (info.has_video) {
578  avcodec_flush_buffers(pCodecCtx);
579  AV_FREE_CONTEXT(pCodecCtx);
580 #if USE_HW_ACCEL
581  if (hw_de_on) {
582  if (hw_device_ctx) {
583  av_buffer_unref(&hw_device_ctx);
584  hw_device_ctx = NULL;
585  }
586  }
587 #endif // USE_HW_ACCEL
588  }
589  if (info.has_audio) {
590  avcodec_flush_buffers(aCodecCtx);
591  AV_FREE_CONTEXT(aCodecCtx);
592  }
593 
594  // Clear final cache
595  final_cache.Clear();
596  working_cache.Clear();
597  missing_frames.Clear();
598 
599  // Clear processed lists
600  {
601  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
602  processed_video_frames.clear();
603  processed_audio_frames.clear();
604  processing_video_frames.clear();
605  processing_audio_frames.clear();
606  missing_audio_frames.clear();
607  missing_video_frames.clear();
608  missing_audio_frames_source.clear();
609  missing_video_frames_source.clear();
610  checked_frames.clear();
611  }
612 
613  // Close the video file
614  avformat_close_input(&pFormatCtx);
615  av_freep(&pFormatCtx);
616 
617  // Reset some variables
618  last_frame = 0;
619  largest_frame_processed = 0;
620  seek_audio_frame_found = 0;
621  seek_video_frame_found = 0;
622  current_video_frame = 0;
623  has_missing_frames = false;
624 
625  last_video_frame.reset();
626  }
627 }
628 
629 bool FFmpegReader::HasAlbumArt() {
630  // Check if the video stream we use is an attached picture
631  // This won't return true if the file has a cover image as a secondary stream
632  // like an MKV file with an attached image file
633  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
634  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
635 }
636 
637 void FFmpegReader::UpdateAudioInfo() {
638  // Set values of FileInfo struct
639  info.has_audio = true;
640  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
641  info.acodec = aCodecCtx->codec->name;
642  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
643  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
644  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
645  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
646  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
647  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
648 
649  // Set audio timebase
650  info.audio_timebase.num = aStream->time_base.num;
651  info.audio_timebase.den = aStream->time_base.den;
652 
653  // Get timebase of audio stream (if valid) and greater than the current duration
654  if (aStream->duration > 0.0f && aStream->duration > info.duration)
655  info.duration = aStream->duration * info.audio_timebase.ToDouble();
656 
657  // Check for an invalid video length
658  if (info.has_video && info.video_length <= 0) {
659  // Calculate the video length from the audio duration
661  }
662 
663  // Set video timebase (if no video stream was found)
664  if (!info.has_video) {
665  // Set a few important default video settings (so audio can be divided into frames)
666  info.fps.num = 24;
667  info.fps.den = 1;
668  info.video_timebase.num = 1;
669  info.video_timebase.den = 24;
671  info.width = 720;
672  info.height = 480;
673  }
674 
675  // Fix invalid video lengths for certain types of files (MP3 for example)
676  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
678  }
679 
680  // Add audio metadata (if any found)
681  AVDictionaryEntry *tag = NULL;
682  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
683  QString str_key = tag->key;
684  QString str_value = tag->value;
685  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
686  }
687 }
688 
689 void FFmpegReader::UpdateVideoInfo() {
690  if (check_fps)
691  // Already initialized all the video metadata, no reason to do it again
692  return;
693 
694  // Set values of FileInfo struct
695  info.has_video = true;
696  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
697  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
698  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
699  info.vcodec = pCodecCtx->codec->name;
700  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
701 
702  // Frame rate from the container and codec
703  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
704  info.fps.num = framerate.num;
705  info.fps.den = framerate.den;
706 
707  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
708 
709  // TODO: remove excessive debug info in the next releases
710  // The debug info below is just for comparison and troubleshooting on users side during the transition period
711  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
712 
713  if (pStream->sample_aspect_ratio.num != 0) {
714  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
715  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
716  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
717  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
718  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
719  } else {
720  info.pixel_ratio.num = 1;
721  info.pixel_ratio.den = 1;
722  }
723  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
724 
725  // Calculate the DAR (display aspect ratio)
727 
728  // Reduce size fraction
729  size.Reduce();
730 
731  // Set the ratio based on the reduced fraction
732  info.display_ratio.num = size.num;
733  info.display_ratio.den = size.den;
734 
735  // Get scan type and order from codec context/params
736  if (!check_interlace) {
737  check_interlace = true;
738  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
739  switch(field_order) {
740  case AV_FIELD_PROGRESSIVE:
741  info.interlaced_frame = false;
742  break;
743  case AV_FIELD_TT:
744  case AV_FIELD_TB:
745  info.interlaced_frame = true;
746  info.top_field_first = true;
747  break;
748  case AV_FIELD_BT:
749  case AV_FIELD_BB:
750  info.interlaced_frame = true;
751  info.top_field_first = false;
752  break;
753  case AV_FIELD_UNKNOWN:
754  // Check again later?
755  check_interlace = false;
756  break;
757  }
758  // check_interlace will prevent these checks being repeated,
759  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
760  }
761 
762  // Set the video timebase
763  info.video_timebase.num = pStream->time_base.num;
764  info.video_timebase.den = pStream->time_base.den;
765 
766  // Set the duration in seconds, and video length (# of frames)
767  info.duration = pStream->duration * info.video_timebase.ToDouble();
768 
769  // Check for valid duration (if found)
770  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
771  // Use the format's duration
772  info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
773 
774  // Calculate duration from filesize and bitrate (if any)
775  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
776  // Estimate from bitrate, total bytes, and framerate
778 
779  // No duration found in stream of file
780  if (info.duration <= 0.0f) {
781  // No duration is found in the video stream
782  info.duration = -1;
783  info.video_length = -1;
784  is_duration_known = false;
785  } else {
786  // Yes, a duration was found
787  is_duration_known = true;
788 
789  // Calculate number of frames
791  }
792 
793  // Override an invalid framerate
794  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
795  // Calculate FPS, duration, video bit rate, and video length manually
796  // by scanning through all the video stream packets
797  CheckFPS();
798  }
799 
800  // Add video metadata (if any)
801  AVDictionaryEntry *tag = NULL;
802  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
803  QString str_key = tag->key;
804  QString str_value = tag->value;
805  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
806  }
807 }
808 
810  return this->is_duration_known;
811 }
812 
813 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
814  // Check for open reader (or throw exception)
815  if (!is_open)
816  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
817 
818  // Adjust for a requested frame that is too small or too large
819  if (requested_frame < 1)
820  requested_frame = 1;
821  if (requested_frame > info.video_length && is_duration_known)
822  requested_frame = info.video_length;
823  if (info.has_video && info.video_length == 0)
824  // Invalid duration of video file
825  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
826 
827  // Debug output
828  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
829 
830  // Check the cache for this frame
831  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
832  if (frame) {
833  // Debug output
834  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
835 
836  // Return the cached frame
837  return frame;
838  } else {
839  // Check the cache a 2nd time (due to a potential previous lock)
840  frame = final_cache.GetFrame(requested_frame);
841  if (frame) {
842  // Debug output
843  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
844 
845  // Return the cached frame
846  } else {
847  // Frame is not in cache
848  // Reset seek count
849  seek_count = 0;
850 
851  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
852  if (last_frame == 0 && requested_frame != 1)
853  // Get first frame
854  ReadStream(1);
855 
856  // Are we within X frames of the requested frame?
857  int64_t diff = requested_frame - last_frame;
858  if (diff >= 1 && diff <= 20) {
859  // Continue walking the stream
860  frame = ReadStream(requested_frame);
861  } else {
862  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
863  if (enable_seek)
864  // Only seek if enabled
865  Seek(requested_frame);
866 
867  else if (!enable_seek && diff < 0) {
868  // Start over, since we can't seek, and the requested frame is smaller than our position
869  Close();
870  Open();
871  }
872 
873  // Then continue walking the stream
874  frame = ReadStream(requested_frame);
875  }
876  }
877  return frame;
878  }
879 }
880 
881 // Read the stream until we find the requested Frame
882 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
883  // Allocate video frame
884  bool end_of_stream = false;
885  bool check_seek = false;
886  bool frame_finished = false;
887  int packet_error = -1;
888 
889  // Minimum number of packets to process (for performance reasons)
890  int packets_processed = 0;
891  int minimum_packets = 1;
892  int max_packets = 4096;
893 
894  // Debug output
895  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames);
896 
897  // Loop through the stream until the correct frame is found
898  while (true) {
899  // Get the next packet into a local variable called packet
900  packet_error = GetNextPacket();
901 
902  int processing_video_frames_size = 0;
903  int processing_audio_frames_size = 0;
904  {
905  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
906  processing_video_frames_size = processing_video_frames.size();
907  processing_audio_frames_size = processing_audio_frames.size();
908  }
909 
910  // Wait if too many frames are being processed
911  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
912  std::this_thread::sleep_for(std::chrono::milliseconds(3));
913  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
914  processing_video_frames_size = processing_video_frames.size();
915  processing_audio_frames_size = processing_audio_frames.size();
916  }
917 
918  // Get the next packet (if any)
919  if (packet_error < 0) {
920  // Break loop when no more packets found
921  end_of_stream = true;
922  break;
923  }
924 
925  // Debug output
926  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
927 
928  // Video packet
929  if (info.has_video && packet->stream_index == videoStream) {
930  // Reset this counter, since we have a video packet
931  num_packets_since_video_frame = 0;
932 
933  // Check the status of a seek (if any)
934  if (is_seeking) {
935  check_seek = CheckSeek(true);
936  } else {
937  check_seek = false;
938  }
939 
940  if (check_seek) {
941  // Jump to the next iteration of this loop
942  continue;
943  }
944 
945  // Packet may become NULL on Close inside Seek if CheckSeek returns false
946  if (!packet) {
947  // Jump to the next iteration of this loop
948  continue;
949  }
950 
951  // Get the AVFrame from the current packet
952  frame_finished = GetAVFrame();
953 
954  // Check if the AVFrame is finished and set it
955  if (frame_finished) {
956  // Update PTS / Frame Offset (if any)
957  UpdatePTSOffset(true);
958 
959  // Process Video Packet
960  ProcessVideoPacket(requested_frame);
961  }
962 
963  }
964  // Audio packet
965  else if (info.has_audio && packet->stream_index == audioStream) {
966  // Increment this (to track # of packets since the last video packet)
967  num_packets_since_video_frame++;
968 
969  // Check the status of a seek (if any)
970  if (is_seeking) {
971  check_seek = CheckSeek(false);
972  } else {
973  check_seek = false;
974  }
975 
976  if (check_seek) {
977  // Jump to the next iteration of this loop
978  continue;
979  }
980 
981  // Packet may become NULL on Close inside Seek if CheckSeek returns false
982  if (!packet) {
983  // Jump to the next iteration of this loop
984  continue;
985  }
986 
987  // Update PTS / Frame Offset (if any)
988  UpdatePTSOffset(false);
989 
990  // Determine related video frame and starting sample # from audio PTS
991  AudioLocation location = GetAudioPTSLocation(packet->pts);
992 
993  // Process Audio Packet
994  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
995  }
996 
997  // Check if working frames are 'finished'
998  if (!is_seeking) {
999  // Check for final frames
1000  CheckWorkingFrames(false, requested_frame);
1001  }
1002 
1003  // Check if requested 'final' frame is available
1004  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1005 
1006  // Increment frames processed
1007  packets_processed++;
1008 
1009  // Break once the frame is found
1010  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
1011  break;
1012 
1013  } // end while
1014 
1015  // Debug output
1016  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count());
1017 
1018  // End of stream?
1019  if (end_of_stream)
1020  // Mark the any other working frames as 'finished'
1021  CheckWorkingFrames(end_of_stream, requested_frame);
1022 
1023  // Return requested frame (if found)
1024  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1025  if (frame)
1026  // Return prepared frame
1027  return frame;
1028  else {
1029 
1030  // Check if largest frame is still cached
1031  frame = final_cache.GetFrame(largest_frame_processed);
1032  if (frame) {
1033  // return the largest processed frame (assuming it was the last in the video file)
1034  return frame;
1035  } else {
1036  // The largest processed frame is no longer in cache, return a blank frame
1037  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1038  f->AddColor(info.width, info.height, "#000");
1039  return f;
1040  }
1041  }
1042 
1043 }
1044 
1045 // Get the next packet (if any)
1046 int FFmpegReader::GetNextPacket() {
1047  int found_packet = 0;
1048  AVPacket *next_packet;
1049  next_packet = new AVPacket();
1050  found_packet = av_read_frame(pFormatCtx, next_packet);
1051 
1052  if (packet) {
1053  // Remove previous packet before getting next one
1054  RemoveAVPacket(packet);
1055  packet = NULL;
1056  }
1057  if (found_packet >= 0) {
1058  // Update current packet pointer
1059  packet = next_packet;
1060  } else {
1061  delete next_packet;
1062  }
1063  // Return if packet was found (or error number)
1064  return found_packet;
1065 }
1066 
1067 // Get an AVFrame (if any)
1068 bool FFmpegReader::GetAVFrame() {
1069  int frameFinished = -1;
1070  int ret = 0;
1071 
1072  // Decode video frame
1073  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1074 
1075 #if IS_FFMPEG_3_2
1076  frameFinished = 0;
1077  ret = avcodec_send_packet(pCodecCtx, packet);
1078 
1079  #if USE_HW_ACCEL
1080  // Get the format from the variables set in get_hw_dec_format
1081  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1082  hw_de_av_device_type = hw_de_av_device_type_global;
1083  #endif // USE_HW_ACCEL
1084  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1085  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)");
1086  }
1087  else {
1088  AVFrame *next_frame2;
1089  #if USE_HW_ACCEL
1090  if (hw_de_on && hw_de_supported) {
1091  next_frame2 = AV_ALLOCATE_FRAME();
1092  }
1093  else
1094  #endif // USE_HW_ACCEL
1095  {
1096  next_frame2 = next_frame;
1097  }
1098  pFrame = AV_ALLOCATE_FRAME();
1099  while (ret >= 0) {
1100  ret = avcodec_receive_frame(pCodecCtx, next_frame2);
1101  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1102  break;
1103  }
1104  if (ret != 0) {
1105  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)");
1106  }
1107  #if USE_HW_ACCEL
1108  if (hw_de_on && hw_de_supported) {
1109  int err;
1110  if (next_frame2->format == hw_de_av_pix_fmt) {
1111  next_frame->format = AV_PIX_FMT_YUV420P;
1112  if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1113  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)");
1114  }
1115  if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1116  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)");
1117  }
1118  }
1119  }
1120  else
1121  #endif // USE_HW_ACCEL
1122  { // No hardware acceleration used -> no copy from GPU memory needed
1123  next_frame = next_frame2;
1124  }
1125 
1126  // TODO also handle possible further frames
1127  // Use only the first frame like avcodec_decode_video2
1128  if (frameFinished == 0 ) {
1129  frameFinished = 1;
1130  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1131  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1132  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1133  }
1134  }
1135  #if USE_HW_ACCEL
1136  if (hw_de_on && hw_de_supported) {
1137  AV_FREE_FRAME(&next_frame2);
1138  }
1139  #endif // USE_HW_ACCEL
1140  }
1141 #else
1142  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1143 
1144  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1145  pFrame = AV_ALLOCATE_FRAME();
1146 
1147  // is frame finished
1148  if (frameFinished) {
1149  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1150  // must make a copy of the image data before this method is called again.
1151  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1152  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1153  info.height);
1154  }
1155 #endif // IS_FFMPEG_3_2
1156 
1157  // deallocate the frame
1158  AV_FREE_FRAME(&next_frame);
1159 
1160  // Did we get a video frame?
1161  return frameFinished;
1162 }
1163 
1164 // Check the current seek position and determine if we need to seek again
1165 bool FFmpegReader::CheckSeek(bool is_video) {
1166  // Are we seeking for a specific frame?
1167  if (is_seeking) {
1168  // Determine if both an audio and video packet have been decoded since the seek happened.
1169  // If not, allow the ReadStream method to keep looping
1170  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1171  return false;
1172 
1173  // Check for both streams
1174  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1175  return false;
1176 
1177  // Determine max seeked frame
1178  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1179  if (seek_video_frame_found > max_seeked_frame)
1180  max_seeked_frame = seek_video_frame_found;
1181 
1182  // determine if we are "before" the requested frame
1183  if (max_seeked_frame >= seeking_frame) {
1184  // SEEKED TOO FAR
1185  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1186 
1187  // Seek again... to the nearest Keyframe
1188  Seek(seeking_frame - (10 * seek_count * seek_count));
1189  } else {
1190  // SEEK WORKED
1191  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1192 
1193  // Seek worked, and we are "before" the requested frame
1194  is_seeking = false;
1195  seeking_frame = 0;
1196  seeking_pts = -1;
1197  }
1198  }
1199 
1200  // return the pts to seek to (if any)
1201  return is_seeking;
1202 }
1203 
1204 // Process a video packet
1205 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1206  // Calculate current frame #
1207  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
1208 
1209  // Track 1st video packet after a successful seek
1210  if (!seek_video_frame_found && is_seeking)
1211  seek_video_frame_found = current_frame;
1212 
1213  // Are we close enough to decode the frame? and is this frame # valid?
1214  if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) {
1215  // Remove frame and packet
1216  RemoveAVFrame(pFrame);
1217 
1218  // Debug output
1219  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame);
1220 
1221  // Skip to next frame without decoding or caching
1222  return;
1223  }
1224 
1225  // Debug output
1226  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1227 
1228  // Init some things local (for OpenMP)
1229  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1230  int height = info.height;
1231  int width = info.width;
1232  int64_t video_length = info.video_length;
1233  AVFrame *my_frame = pFrame;
1234  pFrame = NULL;
1235 
1236  // Add video frame to list of processing video frames
1237  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1238  processing_video_frames[current_frame] = current_frame;
1239 
1240  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1241  AVFrame *pFrameRGB = nullptr;
1242  uint8_t *buffer = nullptr;
1243 
1244  // Allocate an AVFrame structure
1245  pFrameRGB = AV_ALLOCATE_FRAME();
1246  if (pFrameRGB == nullptr)
1247  throw OutOfMemory("Failed to allocate frame buffer", path);
1248 
1249  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1250  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1251  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1252  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1253  // the future.
1254  int max_width = info.width;
1255  int max_height = info.height;
1256 
1257  Clip *parent = (Clip *) ParentClip();
1258  if (parent) {
1259  if (parent->ParentTimeline()) {
1260  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1261  max_width = parent->ParentTimeline()->preview_width;
1262  max_height = parent->ParentTimeline()->preview_height;
1263  }
1264  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1265  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1266  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1267  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1268  max_width = std::max(float(max_width), max_width * max_scale_x);
1269  max_height = std::max(float(max_height), max_height * max_scale_y);
1270 
1271  } else if (parent->scale == SCALE_CROP) {
1272  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1273  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1274  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1275  QSize width_size(max_width * max_scale_x,
1276  round(max_width / (float(info.width) / float(info.height))));
1277  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1278  max_height * max_scale_y);
1279  // respect aspect ratio
1280  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1281  max_width = std::max(max_width, width_size.width());
1282  max_height = std::max(max_height, width_size.height());
1283  } else {
1284  max_width = std::max(max_width, height_size.width());
1285  max_height = std::max(max_height, height_size.height());
1286  }
1287 
1288  } else {
1289  // Scale video to equivalent unscaled size
1290  // Since the preview window can change sizes, we want to always
1291  // scale against the ratio of original video size to timeline size
1292  float preview_ratio = 1.0;
1293  if (parent->ParentTimeline()) {
1294  Timeline *t = (Timeline *) parent->ParentTimeline();
1295  preview_ratio = t->preview_width / float(t->info.width);
1296  }
1297  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1298  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1299  max_width = info.width * max_scale_x * preview_ratio;
1300  max_height = info.height * max_scale_y * preview_ratio;
1301  }
1302  }
1303 
1304  // Determine if image needs to be scaled (for performance reasons)
1305  int original_height = height;
1306  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1307  // Override width and height (but maintain aspect ratio)
1308  float ratio = float(width) / float(height);
1309  int possible_width = round(max_height * ratio);
1310  int possible_height = round(max_width / ratio);
1311 
1312  if (possible_width <= max_width) {
1313  // use calculated width, and max_height
1314  width = possible_width;
1315  height = max_height;
1316  } else {
1317  // use max_width, and calculated height
1318  width = max_width;
1319  height = possible_height;
1320  }
1321  }
1322 
1323  // Determine required buffer size and allocate buffer
1324  const int bytes_per_pixel = 4;
1325  int buffer_size = width * height * bytes_per_pixel;
1326  buffer = new unsigned char[buffer_size]();
1327 
1328  // Copy picture data from one AVFrame (or AVPicture) to another one.
1329  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1330 
1331  int scale_mode = SWS_FAST_BILINEAR;
1332  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1333  scale_mode = SWS_BICUBIC;
1334  }
1335  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1336  height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1337 
1338  // Resize / Convert to RGB
1339  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
1340  original_height, pFrameRGB->data, pFrameRGB->linesize);
1341 
1342  // Create or get the existing frame object
1343  std::shared_ptr<Frame> f = CreateFrame(current_frame);
1344 
1345  // Add Image data to frame
1346  if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) {
1347  // Add image with no alpha channel, Speed optimization
1348  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
1349  } else {
1350  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
1351  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
1352  }
1353 
1354  // Update working cache
1355  working_cache.Add(f);
1356 
1357  // Keep track of last last_video_frame
1358  last_video_frame = f;
1359 
1360  // Free the RGB image
1361  AV_FREE_FRAME(&pFrameRGB);
1362 
1363  // Remove frame and packet
1364  RemoveAVFrame(my_frame);
1365  sws_freeContext(img_convert_ctx);
1366 
1367  // Remove video frame from list of processing video frames
1368  {
1369  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1370  processing_video_frames.erase(current_frame);
1371  processed_video_frames[current_frame] = current_frame;
1372  }
1373 
1374  // Debug output
1375  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number);
1376 }
1377 
1378 // Process an audio packet
1379 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) {
1380  // Track 1st audio packet after a successful seek
1381  if (!seek_audio_frame_found && is_seeking)
1382  seek_audio_frame_found = target_frame;
1383 
1384  // Are we close enough to decode the frame's audio?
1385  if (target_frame < (requested_frame - 20)) {
1386  // Debug output
1387  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1388 
1389  // Skip to next frame without decoding or caching
1390  return;
1391  }
1392 
1393  // Debug output
1394  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1395 
1396  // Init an AVFrame to hold the decoded audio samples
1397  int frame_finished = 0;
1398  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1399  AV_RESET_FRAME(audio_frame);
1400 
1401  int packet_samples = 0;
1402  int data_size = 0;
1403 
1404 #if IS_FFMPEG_3_2
1405  int ret = 0;
1406  frame_finished = 1;
1407  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1408  frame_finished = 0;
1409  ret = avcodec_send_packet(aCodecCtx, packet);
1410  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1411  avcodec_send_packet(aCodecCtx, NULL);
1412  break;
1413  }
1414  if (ret >= 0)
1415  packet->size = 0;
1416  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1417  if (ret >= 0)
1418  frame_finished = 1;
1419  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1420  avcodec_flush_buffers(aCodecCtx);
1421  ret = 0;
1422  }
1423  if (ret >= 0) {
1424  ret = frame_finished;
1425  }
1426  }
1427  if (!packet->data && !frame_finished)
1428  {
1429  ret = -1;
1430  }
1431 #else
1432  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1433 #endif
1434 
1435  if (frame_finished) {
1436 
1437  // determine how many samples were decoded
1438  int plane_size = -1;
1439  data_size = av_samples_get_buffer_size(&plane_size,
1440  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1441  audio_frame->nb_samples,
1442  (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1443 
1444  // Calculate total number of samples
1445  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1446  }
1447 
1448  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1449  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1450 
1451  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1452  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1453  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1454  double sample_seconds = double(pts_total) / info.sample_rate;
1455 
1456  // Debug output
1457  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1458  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1459 
1460  // DEBUG (FOR AUDIO ISSUES)
1461  prev_pts = adjusted_pts;
1462  pts_total += pts_remaining_samples;
1463  pts_counter++;
1464  prev_samples = pts_remaining_samples;
1465 
1466  // Add audio frame to list of processing audio frames
1467  {
1468  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1469  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1470  }
1471 
1472  while (pts_remaining_samples) {
1473  // Get Samples per frame (for this frame number)
1474  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1475 
1476  // Calculate # of samples to add to this frame
1477  int samples = samples_per_frame - previous_packet_location.sample_start;
1478  if (samples > pts_remaining_samples)
1479  samples = pts_remaining_samples;
1480 
1481  // Decrement remaining samples
1482  pts_remaining_samples -= samples;
1483 
1484  if (pts_remaining_samples > 0) {
1485  // next frame
1486  previous_packet_location.frame++;
1487  previous_packet_location.sample_start = 0;
1488 
1489  // Add audio frame to list of processing audio frames
1490  {
1491  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1492  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1493  }
1494 
1495  } else {
1496  // Increment sample start
1497  previous_packet_location.sample_start += samples;
1498  }
1499  }
1500 
1501 
1502  // Allocate audio buffer
1503  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1504 
1505  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16);
1506 
1507  // Create output frame
1508  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1509  AV_RESET_FRAME(audio_converted);
1510  audio_converted->nb_samples = audio_frame->nb_samples;
1511  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1512 
1513  SWRCONTEXT *avr = NULL;
1514  int nb_samples = 0;
1515 
1516  // setup resample context
1517  avr = SWR_ALLOC();
1518  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1519  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1520  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1521  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1522  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1523  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1524  av_opt_set_int(avr, "in_channels", info.channels, 0);
1525  av_opt_set_int(avr, "out_channels", info.channels, 0);
1526  SWR_INIT(avr);
1527 
1528  // Convert audio samples
1529  nb_samples = SWR_CONVERT(avr, // audio resample context
1530  audio_converted->data, // output data pointers
1531  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1532  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1533  audio_frame->data, // input data pointers
1534  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1535  audio_frame->nb_samples); // number of input samples to convert
1536 
1537  // Copy audio samples over original samples
1538  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1539 
1540  // Deallocate resample buffer
1541  SWR_CLOSE(avr);
1542  SWR_FREE(&avr);
1543  avr = NULL;
1544 
1545  // Free AVFrames
1546  av_free(audio_converted->data[0]);
1547  AV_FREE_FRAME(&audio_converted);
1548 
1549  int64_t starting_frame_number = -1;
1550  bool partial_frame = true;
1551  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1552  // Array of floats (to hold samples for each channel)
1553  starting_frame_number = target_frame;
1554  int channel_buffer_size = packet_samples / info.channels;
1555  float *channel_buffer = new float[channel_buffer_size];
1556 
1557  // Init buffer array
1558  for (int z = 0; z < channel_buffer_size; z++)
1559  channel_buffer[z] = 0.0f;
1560 
1561  // Loop through all samples and add them to our Frame based on channel.
1562  // Toggle through each channel number, since channel data is stored like (left right left right)
1563  int channel = 0;
1564  int position = 0;
1565  for (int sample = 0; sample < packet_samples; sample++) {
1566  // Only add samples for current channel
1567  if (channel_filter == channel) {
1568  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1569  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1570 
1571  // Increment audio position
1572  position++;
1573  }
1574 
1575  // increment channel (if needed)
1576  if ((channel + 1) < info.channels)
1577  // move to next channel
1578  channel++;
1579  else
1580  // reset channel
1581  channel = 0;
1582  }
1583 
1584  // Loop through samples, and add them to the correct frames
1585  int start = starting_sample;
1586  int remaining_samples = channel_buffer_size;
1587  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1588  while (remaining_samples > 0) {
1589  // Get Samples per frame (for this frame number)
1590  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1591 
1592  // Calculate # of samples to add to this frame
1593  int samples = samples_per_frame - start;
1594  if (samples > remaining_samples)
1595  samples = remaining_samples;
1596 
1597  // Create or get the existing frame object
1598  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1599 
1600  // Determine if this frame was "partially" filled in
1601  if (samples_per_frame == start + samples)
1602  partial_frame = false;
1603  else
1604  partial_frame = true;
1605 
1606  // Add samples for current channel to the frame.
1607  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 1.0f);
1608 
1609  // Debug output
1610  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1611 
1612  // Add or update cache
1613  working_cache.Add(f);
1614 
1615  // Decrement remaining samples
1616  remaining_samples -= samples;
1617 
1618  // Increment buffer (to next set of samples)
1619  if (remaining_samples > 0)
1620  iterate_channel_buffer += samples;
1621 
1622  // Increment frame number
1623  starting_frame_number++;
1624 
1625  // Reset starting sample #
1626  start = 0;
1627  }
1628 
1629  // clear channel buffer
1630  delete[] channel_buffer;
1631  channel_buffer = NULL;
1632  iterate_channel_buffer = NULL;
1633  }
1634 
1635  // Clean up some arrays
1636  delete[] audio_buf;
1637  audio_buf = NULL;
1638 
1639  // Remove audio frame from list of processing audio frames
1640  {
1641  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1642  // Update all frames as completed
1643  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1644  // Remove the frame # from the processing list. NOTE: If more than one thread is
1645  // processing this frame, the frame # will be in this list multiple times. We are only
1646  // removing a single instance of it here.
1647  processing_audio_frames.erase(processing_audio_frames.find(f));
1648 
1649  // Check and see if this frame is also being processed by another thread
1650  if (processing_audio_frames.count(f) == 0)
1651  // No other thread is processing it. Mark the audio as processed (final)
1652  processed_audio_frames[f] = f;
1653  }
1654 
1655  if (target_frame == starting_frame_number) {
1656  // This typically never happens, but just in case, remove the currently processing number
1657  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1658  }
1659  }
1660 
1661  // Free audio frame
1662  AV_FREE_FRAME(&audio_frame);
1663 
1664  // Debug output
1665  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1);
1666 
1667 }
1668 
1669 
1670 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1671 void FFmpegReader::Seek(int64_t requested_frame) {
1672  // Adjust for a requested frame that is too small or too large
1673  if (requested_frame < 1)
1674  requested_frame = 1;
1675  if (requested_frame > info.video_length)
1676  requested_frame = info.video_length;
1677 
1678  int processing_video_frames_size = 0;
1679  int processing_audio_frames_size = 0;
1680  {
1681  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1682  processing_video_frames_size = processing_video_frames.size();
1683  processing_audio_frames_size = processing_audio_frames.size();
1684  }
1685 
1686  // Debug output
1687  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1688 
1689  // Wait for any processing frames to complete
1690  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1691  std::this_thread::sleep_for(std::chrono::milliseconds(3));
1692  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1693  processing_video_frames_size = processing_video_frames.size();
1694  processing_audio_frames_size = processing_audio_frames.size();
1695  }
1696 
1697  // Clear working cache (since we are seeking to another location in the file)
1698  working_cache.Clear();
1699  missing_frames.Clear();
1700 
1701  // Clear processed lists
1702  {
1703  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1704  processing_audio_frames.clear();
1705  processing_video_frames.clear();
1706  processed_video_frames.clear();
1707  processed_audio_frames.clear();
1708  missing_audio_frames.clear();
1709  missing_video_frames.clear();
1710  missing_audio_frames_source.clear();
1711  missing_video_frames_source.clear();
1712  checked_frames.clear();
1713  }
1714 
1715  // Reset the last frame variable
1716  last_frame = 0;
1717  current_video_frame = 0;
1718  largest_frame_processed = 0;
1719  num_checks_since_final = 0;
1720  num_packets_since_video_frame = 0;
1721  has_missing_frames = false;
1722  bool has_audio_override = info.has_audio;
1723  bool has_video_override = info.has_video;
1724 
1725  // Increment seek count
1726  seek_count++;
1727 
1728  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1729  int buffer_amount = std::max(max_concurrent_frames, 8);
1730  if (requested_frame - buffer_amount < 20) {
1731  // Close and re-open file (basically seeking to frame 1)
1732  Close();
1733  Open();
1734 
1735  // Update overrides (since closing and re-opening might update these)
1736  info.has_audio = has_audio_override;
1737  info.has_video = has_video_override;
1738 
1739  // Not actually seeking, so clear these flags
1740  is_seeking = false;
1741  if (seek_count == 1) {
1742  // Don't redefine this on multiple seek attempts for a specific frame
1743  seeking_frame = 1;
1744  seeking_pts = ConvertFrameToVideoPTS(1);
1745  }
1746  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1747  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1748 
1749  } else {
1750  // Seek to nearest key-frame (aka, i-frame)
1751  bool seek_worked = false;
1752  int64_t seek_target = 0;
1753 
1754  // Seek video stream (if any), except album arts
1755  if (!seek_worked && info.has_video && !HasAlbumArt()) {
1756  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1757  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1758  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1759  } else {
1760  // VIDEO SEEK
1761  is_video_seek = true;
1762  seek_worked = true;
1763  }
1764  }
1765 
1766  // Seek audio stream (if not already seeked... and if an audio stream is found)
1767  if (!seek_worked && info.has_audio) {
1768  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1769  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1770  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1771  } else {
1772  // AUDIO SEEK
1773  is_video_seek = false;
1774  seek_worked = true;
1775  }
1776  }
1777 
1778  // Was the seek successful?
1779  if (seek_worked) {
1780  // Flush audio buffer
1781  if (info.has_audio)
1782  avcodec_flush_buffers(aCodecCtx);
1783 
1784  // Flush video buffer
1785  if (info.has_video)
1786  avcodec_flush_buffers(pCodecCtx);
1787 
1788  // Reset previous audio location to zero
1789  previous_packet_location.frame = -1;
1790  previous_packet_location.sample_start = 0;
1791 
1792  // init seek flags
1793  is_seeking = true;
1794  if (seek_count == 1) {
1795  // Don't redefine this on multiple seek attempts for a specific frame
1796  seeking_pts = seek_target;
1797  seeking_frame = requested_frame;
1798  }
1799  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1800  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1801 
1802  } else {
1803  // seek failed
1804  is_seeking = false;
1805  seeking_pts = 0;
1806  seeking_frame = 0;
1807 
1808  // dislable seeking for this reader (since it failed)
1809  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1810  enable_seek = false;
1811 
1812  // Close and re-open file (basically seeking to frame 1)
1813  Close();
1814  Open();
1815 
1816  // Update overrides (since closing and re-opening might update these)
1817  info.has_audio = has_audio_override;
1818  info.has_video = has_video_override;
1819  }
1820  }
1821 }
1822 
1823 // Get the PTS for the current video packet
1824 int64_t FFmpegReader::GetVideoPTS() {
1825  int64_t current_pts = 0;
1826  if (packet->dts != AV_NOPTS_VALUE)
1827  current_pts = packet->dts;
1828 
1829  // Return adjusted PTS
1830  return current_pts;
1831 }
1832 
1833 // Update PTS Offset (if any)
1834 void FFmpegReader::UpdatePTSOffset(bool is_video) {
1835  // Determine the offset between the PTS and Frame number (only for 1st frame)
1836  if (is_video) {
1837  // VIDEO PACKET
1838  if (video_pts_offset == 99999) // Has the offset been set yet?
1839  {
1840  // Find the difference between PTS and frame number
1841  video_pts_offset = 0 - GetVideoPTS();
1842 
1843  // Find the difference between PTS and frame number
1844  // Also, determine if PTS is invalid (too far away from zero)
1845  // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1846  // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1847  // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1848  // for ideas to improve this logic.
1849  int64_t max_offset = info.video_timebase.Reciprocal().ToFloat();
1850  if (video_pts_offset < -max_offset || video_pts_offset > max_offset) {
1851  // Ignore PTS, it seems invalid
1852  video_pts_offset = 0;
1853  }
1854 
1855  // debug output
1856  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video);
1857  }
1858  } else {
1859  // AUDIO PACKET
1860  if (audio_pts_offset == 99999) // Has the offset been set yet?
1861  {
1862  // Find the difference between PTS and frame number
1863  // Also, determine if PTS is invalid (too far away from zero)
1864  // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1865  // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1866  // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1867  // for ideas to improve this logic.
1868  audio_pts_offset = 0 - packet->pts;
1869  int64_t max_offset = info.audio_timebase.Reciprocal().ToFloat();
1870  if (audio_pts_offset < -max_offset || audio_pts_offset > max_offset) {
1871  // Ignore PTS, it seems invalid
1872  audio_pts_offset = 0;
1873  }
1874 
1875  // debug output
1876  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video);
1877  }
1878  }
1879 }
1880 
1881 // Convert PTS into Frame Number
1882 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
1883  // Apply PTS offset
1884  pts = pts + video_pts_offset;
1885  int64_t previous_video_frame = current_video_frame;
1886 
1887  // Get the video packet start time (in seconds)
1888  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1889 
1890  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1891  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1892 
1893  // Keep track of the expected video frame #
1894  if (current_video_frame == 0)
1895  current_video_frame = frame;
1896  else {
1897 
1898  // Sometimes frames are duplicated due to identical (or similar) timestamps
1899  if (frame == previous_video_frame) {
1900  // return -1 frame number
1901  frame = -1;
1902  } else {
1903  // Increment expected frame
1904  current_video_frame++;
1905  }
1906 
1907  if (current_video_frame < frame)
1908  // has missing frames
1909  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame);
1910 
1911  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1912  // if we are missing a video frame.
1913  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1914  while (current_video_frame < frame) {
1915  if (!missing_video_frames.count(current_video_frame)) {
1916  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame);
1917  missing_video_frames.insert(std::pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1918  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1919  }
1920 
1921  // Mark this reader as containing missing frames
1922  has_missing_frames = true;
1923 
1924  // Increment current frame
1925  current_video_frame++;
1926  }
1927  }
1928 
1929  // Return frame #
1930  return frame;
1931 }
1932 
1933 // Convert Frame Number into Video PTS
1934 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
1935  // Get timestamp of this frame (in seconds)
1936  double seconds = double(frame_number) / info.fps.ToDouble();
1937 
1938  // Calculate the # of video packets in this timestamp
1939  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1940 
1941  // Apply PTS offset (opposite)
1942  return video_pts - video_pts_offset;
1943 }
1944 
1945 // Convert Frame Number into Video PTS
1946 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
1947  // Get timestamp of this frame (in seconds)
1948  double seconds = double(frame_number) / info.fps.ToDouble();
1949 
1950  // Calculate the # of audio packets in this timestamp
1951  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1952 
1953  // Apply PTS offset (opposite)
1954  return audio_pts - audio_pts_offset;
1955 }
1956 
1957 // Calculate Starting video frame and sample # for an audio PTS
1958 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
1959  // Apply PTS offset
1960  pts = pts + audio_pts_offset;
1961 
1962  // Get the audio packet start time (in seconds)
1963  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1964 
1965  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1966  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1967 
1968  // Frame # as a whole number (no more decimals)
1969  int64_t whole_frame = int64_t(frame);
1970 
1971  // Remove the whole number, and only get the decimal of the frame
1972  double sample_start_percentage = frame - double(whole_frame);
1973 
1974  // Get Samples per frame
1975  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1976 
1977  // Calculate the sample # to start on
1978  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1979 
1980  // Protect against broken (i.e. negative) timestamps
1981  if (whole_frame < 1)
1982  whole_frame = 1;
1983  if (sample_start < 0)
1984  sample_start = 0;
1985 
1986  // Prepare final audio packet location
1987  AudioLocation location = {whole_frame, sample_start};
1988 
1989  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1990  if (previous_packet_location.frame != -1) {
1991  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
1992  int64_t orig_frame = location.frame;
1993  int orig_start = location.sample_start;
1994 
1995  // Update sample start, to prevent gaps in audio
1996  location.sample_start = previous_packet_location.sample_start;
1997  location.frame = previous_packet_location.frame;
1998 
1999  // Debug output
2000  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2001 
2002  } else {
2003  // Debug output
2004  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2005 
2006  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2007  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
2008  if (!missing_audio_frames.count(audio_frame)) {
2009  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame);
2010  missing_audio_frames.insert(std::pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
2011  }
2012  }
2013  }
2014  }
2015 
2016  // Set previous location
2017  previous_packet_location = location;
2018 
2019  // Return the associated video frame and starting sample #
2020  return location;
2021 }
2022 
2023 // Create a new Frame (or return an existing one) and add it to the working queue.
2024 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2025  // Check working cache
2026  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2027 
2028  if (!output) {
2029  // Lock
2030  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2031 
2032  // (re-)Check working cache
2033  output = working_cache.GetFrame(requested_frame);
2034  if(output) return output;
2035 
2036  // Create a new frame on the working cache
2037  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2038  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2039  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2040  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2041 
2042  working_cache.Add(output);
2043 
2044  // Set the largest processed frame (if this is larger)
2045  if (requested_frame > largest_frame_processed)
2046  largest_frame_processed = requested_frame;
2047  }
2048  // Return frame
2049  return output;
2050 }
2051 
2052 // Determine if frame is partial due to seek
2053 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2054 
2055  // Sometimes a seek gets partial frames, and we need to remove them
2056  bool seek_trash = false;
2057  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2058  if (seek_video_frame_found > max_seeked_frame) {
2059  max_seeked_frame = seek_video_frame_found;
2060  }
2061  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2062  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2063  seek_trash = true;
2064  }
2065 
2066  return seek_trash;
2067 }
2068 
2069 // Check if a frame is missing and attempt to replace its frame image (and
2070 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) {
2071  // Lock
2072  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2073 
2074  // Increment check count for this frame (or init to 1)
2075  ++checked_frames[requested_frame];
2076 
2077  // Debug output
2078  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_frames[requested_frame]);
2079 
2080  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
2081  std::map<int64_t, int64_t>::iterator itr;
2082  bool found_missing_frame = false;
2083 
2084  // Special MP3 Handling (ignore more than 1 video frame)
2085  if (info.has_audio and info.has_video) {
2086  // If MP3 with single video frame, handle this special case by copying the previously
2087  // decoded image to the new frame. Otherwise, it will spend a huge amount of
2088  // CPU time looking for missing images for all the audio-only frames.
2089  if (checked_frames[requested_frame] > 8 && !missing_video_frames.count(requested_frame) &&
2090  !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
2091  last_video_frame && last_video_frame->has_image_data && HasAlbumArt()) {
2092  missing_video_frames.insert(std::pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
2093  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
2094  missing_frames.Add(last_video_frame);
2095  }
2096  }
2097 
2098  // Check if requested video frame is a missing
2099  if (missing_video_frames.count(requested_frame)) {
2100  int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
2101 
2102  // Increment missing source frame check count (or init to 1)
2103  ++checked_frames[missing_source_frame];
2104 
2105  // Get the previous frame of this missing frame (if it's available in missing cache)
2106  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
2107  if (parent_frame == NULL) {
2108  parent_frame = final_cache.GetFrame(missing_source_frame);
2109  if (parent_frame != NULL) {
2110  // Add missing final frame to missing cache
2111  missing_frames.Add(parent_frame);
2112  }
2113  }
2114 
2115  // Create blank missing frame
2116  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2117 
2118  // Debug output
2119  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2120 
2121  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
2122  if (parent_frame != NULL) {
2123  // Debug output
2124  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2125 
2126  // Add this frame to the processed map (since it's already done)
2127  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
2128  if (parent_image) {
2129  missing_frame->AddImage(std::make_shared<QImage>(*parent_image));
2130  processed_video_frames[missing_frame->number] = missing_frame->number;
2131  }
2132  }
2133  }
2134 
2135  // Check if requested audio frame is a missing
2136  if (missing_audio_frames.count(requested_frame)) {
2137 
2138  // Create blank missing frame
2139  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2140 
2141  // Get Samples per frame (for this frame number)
2142  int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
2143 
2144  // Debug output
2145  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame);
2146 
2147  // Add this frame to the processed map (since it's already done)
2148  missing_frame->AddAudioSilence(samples_per_frame);
2149  processed_audio_frames[missing_frame->number] = missing_frame->number;
2150  }
2151 
2152  return found_missing_frame;
2153 }
2154 
2155 // Check the working queue, and move finished frames to the finished queue
2156 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) {
2157  // Loop through all working queue frames
2158  bool checked_count_tripped = false;
2159  int max_checked_count = 80;
2160 
2161  // Check if requested frame is 'missing'
2162  CheckMissingFrame(requested_frame);
2163 
2164  while (true) {
2165  // Get the front frame of working cache
2166  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
2167 
2168  // Was a frame found?
2169  if (!f)
2170  // No frames found
2171  break;
2172 
2173  // Remove frames which are too old
2174  if (f->number < (requested_frame - (max_concurrent_frames * 2))) {
2175  working_cache.Remove(f->number);
2176  }
2177 
2178  // Check if this frame is 'missing'
2179  CheckMissingFrame(f->number);
2180 
2181  // Init # of times this frame has been checked so far
2182  int checked_count = 0;
2183  int checked_frames_size = 0;
2184 
2185  bool is_video_ready = false;
2186  bool is_audio_ready = false;
2187  { // limit scope of next few lines
2188  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2189  is_video_ready = processed_video_frames.count(f->number);
2190  is_audio_ready = processed_audio_frames.count(f->number);
2191 
2192  // Get check count for this frame
2193  checked_frames_size = checked_frames.size();
2194  if (!checked_count_tripped || f->number >= requested_frame)
2195  checked_count = checked_frames[f->number];
2196  else
2197  // Force checked count over the limit
2198  checked_count = max_checked_count;
2199  }
2200 
2201  if (previous_packet_location.frame == f->number && !end_of_stream)
2202  is_audio_ready = false; // don't finalize the last processed audio frame
2203  bool is_seek_trash = IsPartialFrame(f->number);
2204 
2205  // Adjust for available streams
2206  if (!info.has_video) is_video_ready = true;
2207  if (!info.has_audio) is_audio_ready = true;
2208 
2209  // Make final any frames that get stuck (for whatever reason)
2210  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
2211  // Debug output
2212  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2213 
2214  // Trigger checked count tripped mode (clear out all frames before requested frame)
2215  checked_count_tripped = true;
2216 
2217  if (info.has_video && !is_video_ready && last_video_frame) {
2218  // Copy image from last frame
2219  f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2220  is_video_ready = true;
2221  }
2222 
2223  if (info.has_audio && !is_audio_ready) {
2224  // Mark audio as processed, and indicate the frame has audio data
2225  is_audio_ready = true;
2226  }
2227  }
2228 
2229  // Debug output
2230  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2231 
2232  // Check if working frame is final
2233  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) {
2234  // Debug output
2235  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
2236 
2237  if (!is_seek_trash) {
2238  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
2239  if (info.has_video && !is_video_ready && last_video_frame)
2240  // Copy image from last frame
2241  f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2242 
2243  // Reset counter since last 'final' frame
2244  num_checks_since_final = 0;
2245 
2246  // Move frame to final cache
2247  final_cache.Add(f);
2248 
2249  // Add to missing cache (if another frame depends on it)
2250  {
2251  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2252  if (missing_video_frames_source.count(f->number)) {
2253  // Debug output
2254  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count());
2255  missing_frames.Add(f);
2256  }
2257 
2258  // Remove from 'checked' count
2259  checked_frames.erase(f->number);
2260  }
2261 
2262  // Remove frame from working cache
2263  working_cache.Remove(f->number);
2264 
2265  // Update last frame processed
2266  last_frame = f->number;
2267 
2268  } else {
2269  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2270  working_cache.Remove(f->number);
2271  }
2272 
2273  } else {
2274  // Stop looping
2275  break;
2276  }
2277  }
2278 }
2279 
2280 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2281 void FFmpegReader::CheckFPS() {
2282  check_fps = true;
2283 
2284 
2285  int first_second_counter = 0;
2286  int second_second_counter = 0;
2287  int third_second_counter = 0;
2288  int forth_second_counter = 0;
2289  int fifth_second_counter = 0;
2290  int frames_detected = 0;
2291  int64_t pts = 0;
2292 
2293  // Loop through the stream
2294  while (true) {
2295  // Get the next packet (if any)
2296  if (GetNextPacket() < 0)
2297  // Break loop when no more packets found
2298  break;
2299 
2300  // Video packet
2301  if (packet->stream_index == videoStream) {
2302  // Check if the AVFrame is finished and set it
2303  if (GetAVFrame()) {
2304  // Update PTS / Frame Offset (if any)
2305  UpdatePTSOffset(true);
2306 
2307  // Get PTS of this packet
2308  pts = GetVideoPTS();
2309 
2310  // Remove pFrame
2311  RemoveAVFrame(pFrame);
2312 
2313  // Apply PTS offset
2314  pts += video_pts_offset;
2315 
2316  // Get the video packet start time (in seconds)
2317  double video_seconds = double(pts) * info.video_timebase.ToDouble();
2318 
2319  // Increment the correct counter
2320  if (video_seconds <= 1.0)
2321  first_second_counter++;
2322  else if (video_seconds > 1.0 && video_seconds <= 2.0)
2323  second_second_counter++;
2324  else if (video_seconds > 2.0 && video_seconds <= 3.0)
2325  third_second_counter++;
2326  else if (video_seconds > 3.0 && video_seconds <= 4.0)
2327  forth_second_counter++;
2328  else if (video_seconds > 4.0 && video_seconds <= 5.0)
2329  fifth_second_counter++;
2330 
2331  // Increment counters
2332  frames_detected++;
2333  }
2334  }
2335  }
2336 
2337  // Double check that all counters have greater than zero (or give up)
2338  if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
2339  // Calculate average FPS (average of first few seconds)
2340  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
2341  int avg_fps = round(sum_fps / 4.0f);
2342 
2343  // Update FPS
2344  info.fps = Fraction(avg_fps, 1);
2345 
2346  // Update Duration and Length
2347  info.video_length = frames_detected;
2348  info.duration = frames_detected / (sum_fps / 4.0f);
2349 
2350  // Update video bit rate
2352  } else if (second_second_counter != 0 && third_second_counter != 0) {
2353  // Calculate average FPS (only on second 2)
2354  int sum_fps = second_second_counter;
2355 
2356  // Update FPS
2357  info.fps = Fraction(sum_fps, 1);
2358 
2359  // Update Duration and Length
2360  info.video_length = frames_detected;
2361  info.duration = frames_detected / float(sum_fps);
2362 
2363  // Update video bit rate
2365  } else {
2366  // Too short to determine framerate, just default FPS
2367  // Set a few important default video settings (so audio can be divided into frames)
2368  info.fps.num = 30;
2369  info.fps.den = 1;
2370 
2371  // Calculate number of frames
2372  info.video_length = frames_detected;
2373  info.duration = frames_detected / info.fps.ToFloat();
2374  }
2375 }
2376 
2377 // Remove AVFrame from cache (and deallocate its memory)
2378 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2379  // Remove pFrame (if exists)
2380  if (remove_frame) {
2381  // Free memory
2382  av_freep(&remove_frame->data[0]);
2383 #ifndef WIN32
2384  AV_FREE_FRAME(&remove_frame);
2385 #endif
2386  }
2387 }
2388 
2389 // Remove AVPacket from cache (and deallocate its memory)
2390 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2391  // deallocate memory for packet
2392  AV_FREE_PACKET(remove_packet);
2393 
2394  // Delete the object
2395  delete remove_packet;
2396 }
2397 
2398 /// Get the smallest video frame that is still being processed
2399 int64_t FFmpegReader::GetSmallestVideoFrame() {
2400  // Loop through frame numbers
2401  std::map<int64_t, int64_t>::iterator itr;
2402  int64_t smallest_frame = -1;
2403  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2404  for (itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) {
2405  if (itr->first < smallest_frame || smallest_frame == -1)
2406  smallest_frame = itr->first;
2407  }
2408 
2409  // Return frame number
2410  return smallest_frame;
2411 }
2412 
2413 /// Get the smallest audio frame that is still being processed
2414 int64_t FFmpegReader::GetSmallestAudioFrame() {
2415  // Loop through frame numbers
2416  std::map<int64_t, int64_t>::iterator itr;
2417  int64_t smallest_frame = -1;
2418  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2419  for (itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) {
2420  if (itr->first < smallest_frame || smallest_frame == -1)
2421  smallest_frame = itr->first;
2422  }
2423 
2424  // Return frame number
2425  return smallest_frame;
2426 }
2427 
2428 // Generate JSON string of this object
2429 std::string FFmpegReader::Json() const {
2430 
2431  // Return formatted string
2432  return JsonValue().toStyledString();
2433 }
2434 
2435 // Generate Json::Value for this object
2436 Json::Value FFmpegReader::JsonValue() const {
2437 
2438  // Create root json object
2439  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2440  root["type"] = "FFmpegReader";
2441  root["path"] = path;
2442 
2443  // return JsonValue
2444  return root;
2445 }
2446 
2447 // Load JSON string into this object
2448 void FFmpegReader::SetJson(const std::string value) {
2449 
2450  // Parse JSON string into JSON objects
2451  try {
2452  const Json::Value root = openshot::stringToJson(value);
2453  // Set all values that match
2454  SetJsonValue(root);
2455  }
2456  catch (const std::exception& e) {
2457  // Error parsing JSON (or missing keys)
2458  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2459  }
2460 }
2461 
2462 // Load Json::Value into this object
2463 void FFmpegReader::SetJsonValue(const Json::Value root) {
2464 
2465  // Set parent data
2467 
2468  // Set data from Json (if key is found)
2469  if (!root["path"].isNull())
2470  path = root["path"].asString();
2471 
2472  // Re-Open path, and re-init everything (if needed)
2473  if (is_open) {
2474  Close();
2475  Open();
2476  }
2477 }
Header file for all Exception classes.
AVPixelFormat hw_de_av_pix_fmt_global
AVHWDeviceType hw_de_av_device_type_global
int hw_de_on
Header file for FFmpegReader class.
#define AV_FREE_CONTEXT(av_context)
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_GET_CODEC_TYPE(av_stream)
#define PixelFormat
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
#define AV_FIND_DECODER_CODEC_ID(av_stream)
#define AV_ALLOCATE_FRAME()
#define AV_REGISTER_ALL
#define PIX_FMT_RGBA
#define SWR_FREE(ctx)
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_FREE_PACKET(av_packet)
#define SWRCONTEXT
#define AVCODEC_REGISTER_ALL
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AV_RESET_FRAME(av_frame)
AVDictionary * opts
if(!codec) codec
#define FF_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Header file for Timeline class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
int64_t Count()
Count the frames in the queue.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Remove(int64_t frame_number)
Remove a specific frame.
void Clear()
Clear the cache of all frames.
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:113
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:305
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:306
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:175
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:57
void Open() override
Open File - which is called by the constructor automatically.
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Json::Value JsonValue() const override
Generate Json::Value for this object.
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
virtual ~FFmpegReader()
Destructor.
std::string Json() const override
Generate JSON string of this object.
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
void Close() override
Close File.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a fraction.
Definition: Fraction.h:48
int num
Numerator for the fraction.
Definition: Fraction.h:50
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:54
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:94
int den
Denominator for the fraction.
Definition: Fraction.h:51
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
Exception when no valid codec is found for a file.
Definition: Exceptions.h:158
Exception for files that can not be found or opened.
Definition: Exceptions.h:174
Exception for invalid JSON.
Definition: Exceptions.h:206
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:255
Exception when no streams are found in the file.
Definition: Exceptions.h:270
Exception when memory could not be allocated.
Definition: Exceptions.h:322
Coordinate co
This is the primary coordinate.
Definition: Point.h:84
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
juce::CriticalSection processingCriticalSection
Definition: ReaderBase.h:102
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:171
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:116
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:254
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:338
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:95
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:98
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:92
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:41
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:80
int preview_height
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:58
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:57
This class represents a timeline.
Definition: Timeline.h:168
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:190
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:52
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:55
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:56
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:54
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:60
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:81
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:71
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:86
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:73
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:69
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:80
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:87
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:74
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:72
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:76
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:85
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:66