OpenShot Library | libopenshot  0.7.0
FFmpegWriter.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2025 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <algorithm>
17 #include <iostream>
18 #include <cmath>
19 #include <ctime>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include "FFmpegUtilities.h"
24 
25 #include "FFmpegWriter.h"
26 #include "Exceptions.h"
27 #include "Frame.h"
28 #include "OpenMPUtilities.h"
29 #include "Settings.h"
30 #include "ZmqLogger.h"
31 
32 using namespace openshot;
33 
34 // Multiplexer parameters temporary storage
35 AVDictionary *mux_dict = NULL;
36 
37 #if USE_HW_ACCEL
38 int hw_en_on = 1; // Is set in UI
39 int hw_en_supported = 0; // Is set by FFmpegWriter
40 AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
41 AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
42 static AVBufferRef *hw_device_ctx = NULL;
43 AVFrame *hw_frame = NULL;
44 
45 static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height)
46 {
47  AVBufferRef *hw_frames_ref;
48  AVHWFramesContext *frames_ctx = NULL;
49  int err = 0;
50 
51  if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
52  std::clog << "Failed to create HW frame context.\n";
53  return -1;
54  }
55  frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
56  frames_ctx->format = hw_en_av_pix_fmt;
57  frames_ctx->sw_format = AV_PIX_FMT_NV12;
58  frames_ctx->width = width;
59  frames_ctx->height = height;
60  frames_ctx->initial_pool_size = 20;
61  if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
62  std::clog << "Failed to initialize HW frame context. " <<
63  "Error code: " << av_err2string(err) << "\n";
64  av_buffer_unref(&hw_frames_ref);
65  return err;
66  }
67  ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
68  if (!ctx->hw_frames_ctx)
69  err = AVERROR(ENOMEM);
70 
71  av_buffer_unref(&hw_frames_ref);
72  return err;
73 }
74 #endif // USE_HW_ACCEL
75 
76 FFmpegWriter::FFmpegWriter(const std::string& path) :
77  path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
78  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
79  initial_audio_input_frame_size(0), img_convert_ctx(NULL),
80  video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
81  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
82  write_header(false), write_trailer(false), allow_b_frames(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
83 
84  // Disable audio & video (so they can be independently enabled)
85  info.has_audio = false;
86  info.has_video = false;
87 
88  // Initialize FFMpeg, and register all formats and codecs
89  AV_REGISTER_ALL
90 
91  // auto detect format
92  auto_detect_format();
93 }
94 
95 // Open the writer
96 void FFmpegWriter::Open() {
97  if (!is_open) {
98  // Open the writer
99  is_open = true;
100 
101  // Prepare streams (if needed)
102  if (!prepare_streams)
103  PrepareStreams();
104 
105  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
106  if (info.has_video && video_st)
107  open_video(oc, video_st);
108  if (info.has_audio && audio_st)
109  open_audio(oc, audio_st);
110 
111  // Write header (if needed)
112  if (!write_header)
113  WriteHeader();
114  }
115 }
116 
117 // auto detect format (from path)
118 void FFmpegWriter::auto_detect_format() {
119 
120  // Allocate the output media context
121  AV_OUTPUT_CONTEXT(&oc, path.c_str());
122  if (!oc) {
123  throw OutOfMemory(
124  "Could not allocate memory for AVFormatContext.", path);
125  }
126 
127  // Determine what format to use when encoding this output filename
128  oc->oformat = av_guess_format(NULL, path.c_str(), NULL);
129  if (oc->oformat == nullptr) {
130  throw InvalidFormat("Could not deduce output format from file extension.", path);
131  }
132 
133  // Update video & audio codec name
134  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video) {
135  const AVCodec *vcodec = avcodec_find_encoder(oc->oformat->video_codec);
136  info.vcodec = vcodec ? vcodec->name : std::string();
137  }
138  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio) {
139  const AVCodec *acodec = avcodec_find_encoder(oc->oformat->audio_codec);
140  info.acodec = acodec ? acodec->name : std::string();
141  }
142 }
143 
144 // initialize streams
145 void FFmpegWriter::initialize_streams() {
146  ZmqLogger::Instance()->AppendDebugMethod(
147  "FFmpegWriter::initialize_streams",
148  "oc->oformat->video_codec", oc->oformat->video_codec,
149  "oc->oformat->audio_codec", oc->oformat->audio_codec,
150  "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE);
151 
152  // Add the audio and video streams using the default format codecs and initialize the codecs
153  video_st = NULL;
154  audio_st = NULL;
155  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video)
156  // Add video stream
157  video_st = add_video_stream();
158 
159  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
160  // Add audio stream
161  audio_st = add_audio_stream();
162 }
163 
164 // Set video export options
165 void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) {
166  // Set the video options
167  if (codec.length() > 0) {
168  const AVCodec *new_codec;
169  // Check if the codec selected is a hardware accelerated codec
170 #if USE_HW_ACCEL
171 #if defined(__linux__)
172  if (strstr(codec.c_str(), "_vaapi") != NULL) {
173  new_codec = avcodec_find_encoder_by_name(codec.c_str());
174  hw_en_on = 1;
175  hw_en_supported = 1;
176  hw_en_av_pix_fmt = AV_PIX_FMT_VAAPI;
177  hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
178  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
179  new_codec = avcodec_find_encoder_by_name(codec.c_str());
180  hw_en_on = 1;
181  hw_en_supported = 1;
182  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
183  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
184  } else {
185  new_codec = avcodec_find_encoder_by_name(codec.c_str());
186  hw_en_on = 0;
187  hw_en_supported = 0;
188  }
189 #elif defined(_WIN32)
190  if (strstr(codec.c_str(), "_dxva2") != NULL) {
191  new_codec = avcodec_find_encoder_by_name(codec.c_str());
192  hw_en_on = 1;
193  hw_en_supported = 1;
194  hw_en_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD;
195  hw_en_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
196  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
197  new_codec = avcodec_find_encoder_by_name(codec.c_str());
198  hw_en_on = 1;
199  hw_en_supported = 1;
200  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
201  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
202  } else {
203  new_codec = avcodec_find_encoder_by_name(codec.c_str());
204  hw_en_on = 0;
205  hw_en_supported = 0;
206  }
207 #elif defined(__APPLE__)
208  if (strstr(codec.c_str(), "_videotoolbox") != NULL) {
209  new_codec = avcodec_find_encoder_by_name(codec.c_str());
210  hw_en_on = 1;
211  hw_en_supported = 1;
212  hw_en_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
213  hw_en_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
214  } else {
215  new_codec = avcodec_find_encoder_by_name(codec.c_str());
216  hw_en_on = 0;
217  hw_en_supported = 0;
218  }
219 #else // unknown OS
220  new_codec = avcodec_find_encoder_by_name(codec.c_str());
221 #endif //__linux__/_WIN32/__APPLE__
222 #else // USE_HW_ACCEL
223  new_codec = avcodec_find_encoder_by_name(codec.c_str());
224 #endif // USE_HW_ACCEL
225  if (new_codec == NULL)
226  throw InvalidCodec("A valid video codec could not be found for this file.", path);
227  else {
228  // Set video codec
229  info.vcodec = new_codec->name;
230  }
231  }
232  if (fps.num > 0) {
233  // Set frames per second (if provided)
234  info.fps.num = fps.num;
235  info.fps.den = fps.den;
236 
237  // Set the timebase (inverse of fps)
238  info.video_timebase.num = info.fps.den;
239  info.video_timebase.den = info.fps.num;
240  }
241  if (width >= 1)
242  info.width = width;
243  if (height >= 1)
244  info.height = height;
245  if (pixel_ratio.num > 0) {
246  info.pixel_ratio.num = pixel_ratio.num;
247  info.pixel_ratio.den = pixel_ratio.den;
248  }
249  if (bit_rate >= 1000) // bit_rate is the bitrate in b/s
250  info.video_bit_rate = bit_rate;
251  if ((bit_rate >= 0) && (bit_rate < 256)) // bit_rate is the bitrate in crf
252  info.video_bit_rate = bit_rate;
253 
254  info.interlaced_frame = interlaced;
255  info.top_field_first = top_field_first;
256 
257  // Calculate the DAR (display aspect ratio)
258  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
259 
260  // Reduce size fraction
261  size.Reduce();
262 
263  // Set the ratio based on the reduced fraction
264  info.display_ratio.num = size.num;
265  info.display_ratio.den = size.den;
266 
267  ZmqLogger::Instance()->AppendDebugMethod(
268  "FFmpegWriter::SetVideoOptions (" + codec + ")",
269  "width", width, "height", height,
270  "size.num", size.num, "size.den", size.den,
271  "fps.num", fps.num, "fps.den", fps.den);
272 
273  // Enable / Disable video
274  info.has_video = has_video;
275 }
276 
277 // Set video export options (overloaded function)
278 void FFmpegWriter::SetVideoOptions(std::string codec, int width, int height, Fraction fps, int bit_rate) {
279  // Call full signature with some default parameters
280  FFmpegWriter::SetVideoOptions(
281  true, codec, fps, width, height,
282  openshot::Fraction(1, 1), false, true, bit_rate
283  );
284 }
285 
286 
287 // Set audio export options
288 void FFmpegWriter::SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) {
289  // Set audio options
290  if (codec.length() > 0) {
291  const AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
292  if (new_codec == NULL)
293  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
294  else {
295  // Set audio codec
296  info.acodec = new_codec->name;
297  }
298  }
299  if (sample_rate > 7999)
300  info.sample_rate = sample_rate;
301  if (channels > 0)
302  info.channels = channels;
303  if (bit_rate > 999)
304  info.audio_bit_rate = bit_rate;
305  info.channel_layout = channel_layout;
306 
307  // init resample options (if zero)
308  if (original_sample_rate == 0)
309  original_sample_rate = info.sample_rate;
310  if (original_channels == 0)
311  original_channels = info.channels;
312 
313  ZmqLogger::Instance()->AppendDebugMethod(
314  "FFmpegWriter::SetAudioOptions (" + codec + ")",
315  "sample_rate", sample_rate,
316  "channels", channels,
317  "bit_rate", bit_rate);
318 
319  // Enable / Disable audio
320  info.has_audio = has_audio;
321 }
322 
323 
324 // Set audio export options (overloaded function)
325 void FFmpegWriter::SetAudioOptions(std::string codec, int sample_rate, int bit_rate) {
326  // Call full signature with some default parameters
327  FFmpegWriter::SetAudioOptions(
328  true, codec, sample_rate, 2,
329  openshot::LAYOUT_STEREO, bit_rate
330  );
331 }
332 
333 
334 // Set custom options (some codecs accept additional params)
335 void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string value) {
336  // Declare codec context
337  AVCodecContext *c = NULL;
338  AVStream *st = NULL;
339  std::stringstream convert(value);
340 
341  if (info.has_video && stream == VIDEO_STREAM && video_st) {
342  st = video_st;
343  // Get codec context
344  c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec_ctx);
345  // Was a codec / stream found?
346  if (c) {
347  if (info.interlaced_frame) {
348  c->field_order = info.top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
349  // We only use these two version and ignore AV_FIELD_TB and AV_FIELD_BT
350  // Otherwise we would need to change the whole export window
351  }
352  }
353  } else if (info.has_audio && stream == AUDIO_STREAM && audio_st) {
354  st = audio_st;
355  // Get codec context
356  c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec_ctx);
357  } else
358  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
359 
360  // Init AVOption
361  const AVOption *option = NULL;
362 
363  // Was a codec / stream found?
364  if (c)
365  // Find AVOption (if it exists)
366  option = AV_OPTION_FIND(c->priv_data, name.c_str());
367 
368  // Was option found?
369  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
370  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" ||
371  name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp" || name == "allow_b_frames")) {
372  // Check for specific named options
373  if (name == "g")
374  // Set gop_size
375  convert >> c->gop_size;
376 
377  else if (name == "qmin")
378  // Minimum quantizer
379  convert >> c->qmin;
380 
381  else if (name == "qmax")
382  // Maximum quantizer
383  convert >> c->qmax;
384 
385  else if (name == "max_b_frames")
386  // Maximum number of B-frames between non-B-frames
387  convert >> c->max_b_frames;
388 
389  else if (name == "allow_b_frames")
390  // Preserve configured B-frames for codecs that support them.
391  // Values: 1/true/yes/on to enable, everything else disables.
392  allow_b_frames = (value == "1" || value == "true" || value == "yes" || value == "on");
393 
394  else if (name == "mb_decision")
395  // Macroblock decision mode
396  convert >> c->mb_decision;
397 
398  else if (name == "level")
399  // Set codec level
400  convert >> c->level;
401 
402  else if (name == "profile")
403  // Set codec profile
404  convert >> c->profile;
405 
406  else if (name == "slices")
407  // Indicates number of picture subdivisions
408  convert >> c->slices;
409 
410  else if (name == "rc_min_rate")
411  // Minimum bitrate
412  convert >> c->rc_min_rate;
413 
414  else if (name == "rc_max_rate")
415  // Maximum bitrate
416  convert >> c->rc_max_rate;
417 
418  else if (name == "rc_buffer_size")
419  // Buffer size
420  convert >> c->rc_buffer_size;
421 
422  else if (name == "cqp") {
423  // encode quality and special settings like lossless
424 #if USE_HW_ACCEL
425  if (hw_en_on) {
426  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
427  } else
428 #endif // USE_HW_ACCEL
429  {
430  switch (c->codec_id) {
431 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
432  // FFmpeg 4.0+
433  case AV_CODEC_ID_AV1 :
434  c->bit_rate = 0;
435  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
436  break;
437 #endif
438  case AV_CODEC_ID_VP8 :
439  c->bit_rate = 10000000;
440  av_opt_set_int(c->priv_data, "qp", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
441  break;
442  case AV_CODEC_ID_VP9 :
443  c->bit_rate = 0; // Must be zero!
444  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
445  if (std::stoi(value) == 0) {
446  av_opt_set(c->priv_data, "preset", "veryslow", 0);
447  av_opt_set_int(c->priv_data, "lossless", 1, 0);
448  }
449  break;
450  case AV_CODEC_ID_H264 :
451  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
452  if (std::stoi(value) == 0) {
453  av_opt_set(c->priv_data, "preset", "veryslow", 0);
454  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
455  }
456  break;
457  case AV_CODEC_ID_HEVC :
458  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
459  if (std::stoi(value) == 0) {
460  av_opt_set(c->priv_data, "preset", "veryslow", 0);
461  av_opt_set_int(c->priv_data, "lossless", 1, 0);
462  }
463  break;
464  default:
465  // For all other codecs assume a range of 0-63
466  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
467  c->bit_rate = 0;
468  }
469  }
470  } else if (name == "crf") {
471  // encode quality and special settings like lossless
472 #if USE_HW_ACCEL
473  if (hw_en_on) {
474  double mbs = 15000000.0;
475  if (info.video_bit_rate > 0) {
476  if (info.video_bit_rate > 42) {
477  mbs = 380000.0;
478  }
479  else {
480  mbs *= std::pow(0.912,info.video_bit_rate);
481  }
482  }
483  c->bit_rate = (int)(mbs);
484  } else
485 #endif // USE_HW_ACCEL
486  {
487  switch (c->codec_id) {
488 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
489  // FFmpeg 4.0+
490  case AV_CODEC_ID_AV1 :
491  c->bit_rate = 0;
492  // AV1 only supports "crf" quality values
493  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
494  break;
495 #endif
496  case AV_CODEC_ID_VP8 :
497  c->bit_rate = 10000000;
498  av_opt_set_int(c->priv_data, "crf", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
499  break;
500  case AV_CODEC_ID_VP9 :
501  c->bit_rate = 0; // Must be zero!
502  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 63), 0); // 0-63
503  if (std::stoi(value) == 0) {
504  av_opt_set(c->priv_data, "preset", "veryslow", 0);
505  av_opt_set_int(c->priv_data, "lossless", 1, 0);
506  }
507  break;
508  case AV_CODEC_ID_H264 :
509  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
510  if (std::stoi(value) == 0) {
511  av_opt_set(c->priv_data, "preset", "veryslow", 0);
512  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
513  }
514  break;
515  case AV_CODEC_ID_HEVC :
516  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
517  av_opt_set_int(c->priv_data, "preset", 7, 0);
518  av_opt_set_int(c->priv_data, "forced-idr",1,0);
519  av_opt_set_int(c->priv_data, "qp",std::min(std::stoi(value), 51),0);
520  }
521  else {
522  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
523  }
524  if (std::stoi(value) == 0) {
525  av_opt_set(c->priv_data, "preset", "veryslow", 0);
526  av_opt_set_int(c->priv_data, "lossless", 1, 0);
527  }
528  break;
529  default:
530  // If this codec doesn't support crf calculate a bitrate
531  // TODO: find better formula
532  double mbs = 15000000.0;
533  if (info.video_bit_rate > 0) {
534  if (info.video_bit_rate > 42) {
535  mbs = 380000.0;
536  } else {
537  mbs *= std::pow(0.912, info.video_bit_rate);
538  }
539  }
540  c->bit_rate = (int) (mbs);
541  }
542  }
543  } else if (name == "qp") {
544  // encode quality and special settings like lossless
545 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
546  // FFmpeg 4.0+
547  switch (c->codec_id) {
548  case AV_CODEC_ID_AV1 :
549  c->bit_rate = 0;
550  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
551  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0);
552  }
553  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
554  // Set number of tiles to a fixed value
555  // TODO Let user choose number of tiles
556  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),255), 0);
557  }
558  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
559  // Set number of tiles to a fixed value
560  // TODO Let user choose number of tiles
561  // libaom doesn't have qp only crf
562  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
563  }
564  else {
565  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
566  }
567  case AV_CODEC_ID_HEVC :
568  c->bit_rate = 0;
569  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
570  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),51), 0);
571  av_opt_set_int(c->priv_data, "preset", 7, 0);
572  av_opt_set_int(c->priv_data, "forced-idr",1,0);
573  }
574  break;
575  }
576 #endif // FFmpeg 4.0+
577  } else {
578  // Set AVOption
579  AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c);
580  }
581 
582  ZmqLogger::Instance()->AppendDebugMethod(
583  "FFmpegWriter::SetOption (" + (std::string)name + ")",
584  "stream == VIDEO_STREAM", stream == VIDEO_STREAM);
585 
586  // Muxing dictionary is not part of the codec context.
587  // Just reusing SetOption function to set popular multiplexing presets.
588  } else if (name == "muxing_preset") {
589  if (value == "mp4_faststart") {
590  // 'moov' box to the beginning; only for MOV, MP4
591  av_dict_set(&mux_dict, "movflags", "faststart", 0);
592  } else if (value == "mp4_fragmented") {
593  // write selfcontained fragmented file, minimum length of the fragment 8 sec; only for MOV, MP4
594  av_dict_set(&mux_dict, "movflags", "frag_keyframe", 0);
595  av_dict_set(&mux_dict, "min_frag_duration", "8000000", 0);
596  }
597  } else {
598  throw InvalidOptions("The option is not valid for this codec.", path);
599  }
600 
601 }
602 
604 bool FFmpegWriter::IsValidCodec(std::string codec_name) {
605  // Initialize FFMpeg, and register all formats and codecs
606  AV_REGISTER_ALL
607 
608  // Find the codec (if any)
609  return avcodec_find_encoder_by_name(codec_name.c_str()) != NULL;
610 }
611 
612 // Prepare & initialize streams and open codecs
613 void FFmpegWriter::PrepareStreams() {
614  if (!info.has_audio && !info.has_video)
615  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
616 
617  ZmqLogger::Instance()->AppendDebugMethod(
618  "FFmpegWriter::PrepareStreams [" + path + "]",
619  "info.has_audio", info.has_audio,
620  "info.has_video", info.has_video);
621 
622  // Initialize the streams (i.e. add the streams)
623  initialize_streams();
624 
625  // Mark as 'prepared'
626  prepare_streams = true;
627 }
628 
629 // Write the file header (after the options are set)
630 void FFmpegWriter::WriteHeader() {
631  if (!info.has_audio && !info.has_video)
632  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
633 
634  // Open the output file, if needed
635  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
636  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
637  throw InvalidFile("Could not open or write file.", path);
638  }
639 
640  // Force the output filename (which doesn't always happen for some reason)
641  AV_SET_FILENAME(oc, path.c_str());
642 
643  // Add general metadata (if any)
644  for (auto iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
645  av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0);
646  }
647 
648  // Set multiplexing parameters (only for MP4/MOV containers)
649  AVDictionary *dict = NULL;
650  if (mux_dict) {
651  av_dict_copy(&dict, mux_dict, 0);
652  }
653 
654  // Write the stream header
655  if (avformat_write_header(oc, &dict) != 0) {
656  ZmqLogger::Instance()->AppendDebugMethod(
657  "FFmpegWriter::WriteHeader (avformat_write_header)");
658  throw InvalidFile("Could not write header to file.", path);
659  };
660 
661  // Free multiplexing dictionaries sets
662  if (dict) av_dict_free(&dict);
663  if (mux_dict) av_dict_free(&mux_dict);
664 
665  // Mark as 'written'
666  write_header = true;
667 
668  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader");
669 }
670 
671 // Add a frame to the queue waiting to be encoded.
672 void FFmpegWriter::WriteFrame(std::shared_ptr<openshot::Frame> frame) {
673  // Check for open reader (or throw exception)
674  if (!is_open)
675  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
676 
677  ZmqLogger::Instance()->AppendDebugMethod(
678  "FFmpegWriter::WriteFrame",
679  "frame->number", frame->number,
680  "is_writing", is_writing);
681 
682  // Write frames to video file
683  write_frame(frame);
684 
685  // Keep track of the last frame added
686  last_frame = frame;
687 }
688 
689 // Write all frames in the queue to the video file.
690 void FFmpegWriter::write_frame(std::shared_ptr<Frame> frame) {
691  // Flip writing flag
692  is_writing = true;
693 
694  // Create blank exception
695  bool has_error_encoding_video = false;
696 
697  // Process audio frame
698  if (info.has_audio && audio_st)
699  write_audio_packets(false, frame);
700 
701  // Process video frame
702  if (info.has_video && video_st)
703  process_video_packet(frame);
704 
705  if (info.has_video && video_st) {
706  // Does this frame's AVFrame still exist
707  if (av_frames.count(frame)) {
708  // Get AVFrame
709  AVFrame *frame_final = av_frames[frame];
710 
711  // Write frame to video file
712  if (!write_video_packet(frame, frame_final)) {
713  has_error_encoding_video = true;
714  }
715 
716  // Deallocate buffer and AVFrame
717  av_freep(&(frame_final->data[0]));
718  AV_FREE_FRAME(&frame_final);
719  av_frames.erase(frame);
720  }
721  }
722 
723  // Done writing
724  is_writing = false;
725 
726  // Raise exception from main thread
727  if (has_error_encoding_video)
728  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
729 }
730 
731 // Write a block of frames from a reader
732 void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) {
733  ZmqLogger::Instance()->AppendDebugMethod(
734  "FFmpegWriter::WriteFrame (from Reader)",
735  "start", start,
736  "length", length);
737 
738  // Loop through each frame (and encoded it)
739  for (int64_t number = start; number <= length; number++) {
740  // Get the frame
741  std::shared_ptr<Frame> f = reader->GetFrame(number);
742 
743  // Encode frame
744  WriteFrame(f);
745  }
746 }
747 
748 // Write the file trailer (after all frames are written)
749 void FFmpegWriter::WriteTrailer() {
750  // Process final audio frame (if any)
751  if (info.has_audio && audio_st)
752  write_audio_packets(true, NULL);
753 
754  // Flush encoders (who sometimes hold on to frames)
755  flush_encoders();
756 
757  /* write the trailer, if any. The trailer must be written
758  * before you close the CodecContexts open when you wrote the
759  * header; otherwise write_trailer may try to use memory that
760  * was freed on av_codec_close() */
761  av_write_trailer(oc);
762 
763  // Mark as 'written'
764  write_trailer = true;
765 
766  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer");
767 }
768 
769 // Flush encoders
770 void FFmpegWriter::flush_encoders() {
771  if (info.has_audio && audio_codec_ctx && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec_ctx)->frame_size <= 1)
772  return;
773 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
774  // FFmpeg < 4.0
775  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
776  return;
777 #else
778  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
779  return;
780 #endif
781 
782  // FLUSH VIDEO ENCODER
783  if (info.has_video) {
784  for (;;) {
785 
786  // Increment PTS (in frames and scaled to the codec's timebase)
787  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
788 
789 #if IS_FFMPEG_3_2
790  AVPacket* pkt = av_packet_alloc();
791 #else
792  AVPacket* pkt;
793  av_init_packet(pkt);
794 #endif
795  pkt->data = NULL;
796  pkt->size = 0;
797 
798  /* encode the image */
799  int got_packet = 0;
800  int error_code = 0;
801 
802 #if IS_FFMPEG_3_2
803  // Encode video packet (latest version of FFmpeg)
804  error_code = avcodec_send_frame(video_codec_ctx, NULL);
805  got_packet = 0;
806  while (error_code >= 0) {
807  error_code = avcodec_receive_packet(video_codec_ctx, pkt);
808  if (error_code == AVERROR(EAGAIN)|| error_code == AVERROR_EOF) {
809  got_packet = 0;
810  // Write packet
811  avcodec_flush_buffers(video_codec_ctx);
812  break;
813  }
814  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
815  pkt->stream_index = video_st->index;
816  error_code = av_interleaved_write_frame(oc, pkt);
817  }
818 #else // IS_FFMPEG_3_2
819 
820  // Encode video packet (older than FFmpeg 3.2)
821  error_code = avcodec_encode_video2(video_codec_ctx, pkt, NULL, &got_packet);
822 
823 #endif // IS_FFMPEG_3_2
824 
825  if (error_code < 0) {
826  ZmqLogger::Instance()->AppendDebugMethod(
827  "FFmpegWriter::flush_encoders ERROR ["
828  + av_err2string(error_code) + "]",
829  "error_code", error_code);
830  }
831  if (!got_packet) {
832  break;
833  }
834 
835  // set the timestamp
836  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
837  pkt->stream_index = video_st->index;
838 
839  // Write packet
840  error_code = av_interleaved_write_frame(oc, pkt);
841  if (error_code < 0) {
842  ZmqLogger::Instance()->AppendDebugMethod(
843  "FFmpegWriter::flush_encoders ERROR ["
844  + av_err2string(error_code) + "]",
845  "error_code", error_code);
846  }
847  }
848  }
849 
850  // FLUSH AUDIO ENCODER
851  if (info.has_audio) {
852  for (;;) {
853 #if IS_FFMPEG_3_2
854  AVPacket* pkt = av_packet_alloc();
855 #else
856  AVPacket* pkt;
857  av_init_packet(pkt);
858 #endif
859  pkt->data = NULL;
860  pkt->size = 0;
861  pkt->pts = pkt->dts = audio_timestamp;
862 
863  /* encode the image */
864  int error_code = 0;
865  int got_packet = 0;
866 #if IS_FFMPEG_3_2
867  error_code = avcodec_send_frame(audio_codec_ctx, NULL);
868 #else
869  error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, NULL, &got_packet);
870 #endif
871  if (error_code < 0) {
872  ZmqLogger::Instance()->AppendDebugMethod(
873  "FFmpegWriter::flush_encoders ERROR ["
874  + av_err2string(error_code) + "]",
875  "error_code", error_code);
876  }
877  if (!got_packet) {
878  break;
879  }
880 
881  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
882  // but it fixes lots of PTS related issues when I do this.
883  pkt->pts = pkt->dts = audio_timestamp;
884 
885  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
886  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
887 
888  // set stream
889  pkt->stream_index = audio_st->index;
890  pkt->flags |= AV_PKT_FLAG_KEY;
891 
892  // Write packet
893  error_code = av_interleaved_write_frame(oc, pkt);
894  if (error_code < 0) {
895  ZmqLogger::Instance()->AppendDebugMethod(
896  "FFmpegWriter::flush_encoders ERROR ["
897  + av_err2string(error_code) + "]",
898  "error_code", error_code);
899  }
900 
901  // Increment PTS by duration of packet
902  audio_timestamp += pkt->duration;
903 
904  // deallocate memory for packet
905  AV_FREE_PACKET(pkt);
906  }
907  }
908 
909 }
910 
911 // Close the video codec
912 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
913 {
914 #if USE_HW_ACCEL
915  if (hw_en_on && hw_en_supported) {
916  if (hw_device_ctx) {
917  av_buffer_unref(&hw_device_ctx);
918  hw_device_ctx = NULL;
919  }
920  }
921 #endif // USE_HW_ACCEL
922 
923  // Free any previous memory allocations
924  if (video_codec_ctx != nullptr) {
925  AV_FREE_CONTEXT(video_codec_ctx);
926  av_free(video_codec_ctx);
927  }
928 }
929 
930 // Close the audio codec
931 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
932 {
933  // Clear buffers
934  delete[] samples;
935  delete[] audio_outbuf;
936  delete[] audio_encoder_buffer;
937  samples = NULL;
938  audio_outbuf = NULL;
939  audio_encoder_buffer = NULL;
940 
941  // Deallocate resample buffer
942  if (avr) {
943  SWR_CLOSE(avr);
944  SWR_FREE(&avr);
945  avr = NULL;
946  }
947 
948  if (avr_planar) {
949  SWR_CLOSE(avr_planar);
950  SWR_FREE(&avr_planar);
951  avr_planar = NULL;
952  }
953 
954  // Free any previous memory allocations
955  if (audio_codec_ctx != nullptr) {
956  AV_FREE_CONTEXT(audio_codec_ctx);
957  av_free(audio_codec_ctx);
958  }
959 }
960 
961 // Close the writer
962 void FFmpegWriter::Close() {
963  // Write trailer (if needed)
964  if (!write_trailer)
965  WriteTrailer();
966 
967  // Close each codec
968  if (video_st)
969  close_video(oc, video_st);
970  if (audio_st)
971  close_audio(oc, audio_st);
972 
973  // Remove single software scaler
974  if (img_convert_ctx)
975  sws_freeContext(img_convert_ctx);
976 
977  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
978  /* close the output file */
979  avio_close(oc->pb);
980  }
981 
982  // Reset frame counters
983  video_timestamp = 0;
984  audio_timestamp = 0;
985 
986  // Free the context which frees the streams too
987  avformat_free_context(oc);
988  oc = NULL;
989 
990  // Close writer
991  is_open = false;
992  prepare_streams = false;
993  write_header = false;
994  write_trailer = false;
995 
996  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close");
997 }
998 
999 // Add an AVFrame to the cache
1000 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame *av_frame) {
1001  // Add AVFrame to map (if it does not already exist)
1002  if (!av_frames.count(frame)) {
1003  // Add av_frame
1004  av_frames[frame] = av_frame;
1005  } else {
1006  // Do not add, and deallocate this AVFrame
1007  AV_FREE_FRAME(&av_frame);
1008  }
1009 }
1010 
1011 // Add an audio output stream
1012 AVStream *FFmpegWriter::add_audio_stream() {
1013  // Find the audio codec
1014  const AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1015  if (codec == NULL)
1016  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
1017 
1018  // Free any previous memory allocations
1019  if (audio_codec_ctx != nullptr) {
1020  AV_FREE_CONTEXT(audio_codec_ctx);
1021  }
1022 
1023  // Create a new audio stream
1024  AVStream* st = avformat_new_stream(oc, codec);
1025  if (!st)
1026  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
1027 
1028  // Allocate a new codec context for the stream
1029  ALLOC_CODEC_CTX(audio_codec_ctx, codec, st)
1030 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1031  st->codecpar->codec_id = codec->id;
1032 #endif
1033  AVCodecContext* c = audio_codec_ctx;
1034 
1035  c->codec_id = codec->id;
1036  c->codec_type = AVMEDIA_TYPE_AUDIO;
1037 
1038  // Set the sample parameters
1039  c->bit_rate = info.audio_bit_rate;
1040 #if !HAVE_CH_LAYOUT
1041  c->channels = info.channels;
1042 #endif
1043 
1044  // Set valid sample rate (or throw error)
1045  if (codec->supported_samplerates) {
1046  int i;
1047  for (i = 0; codec->supported_samplerates[i] != 0; i++)
1048  if (info.sample_rate == codec->supported_samplerates[i]) {
1049  // Set the valid sample rate
1050  c->sample_rate = info.sample_rate;
1051  break;
1052  }
1053  if (codec->supported_samplerates[i] == 0)
1054  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
1055  } else
1056  // Set sample rate
1057  c->sample_rate = info.sample_rate;
1058 
1059  uint64_t channel_layout = info.channel_layout;
1060 #if HAVE_CH_LAYOUT
1061  // Set a valid number of channels (or throw error)
1062  AVChannelLayout ch_layout;
1063  av_channel_layout_from_mask(&ch_layout, info.channel_layout);
1064  if (codec->ch_layouts) {
1065  int i;
1066  for (i = 0; av_channel_layout_check(&codec->ch_layouts[i]); i++)
1067  if (av_channel_layout_compare(&ch_layout, &codec->ch_layouts[i])) {
1068  // Set valid channel layout
1069  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1070  break;
1071  }
1072  if (!av_channel_layout_check(&codec->ch_layouts[i]))
1073  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1074  } else
1075  // Set valid channel layout
1076  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1077 #else
1078  // Set a valid number of channels (or throw error)
1079  if (codec->channel_layouts) {
1080  int i;
1081  for (i = 0; codec->channel_layouts[i] != 0; i++)
1082  if (channel_layout == codec->channel_layouts[i]) {
1083  // Set valid channel layout
1084  c->channel_layout = channel_layout;
1085  break;
1086  }
1087  if (codec->channel_layouts[i] == 0)
1088  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1089  } else
1090  // Set valid channel layout
1091  c->channel_layout = channel_layout;
1092 #endif
1093 
1094  // Choose a valid sample_fmt
1095  if (codec->sample_fmts) {
1096  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
1097  // Set sample format to 1st valid format (and then exit loop)
1098  c->sample_fmt = codec->sample_fmts[i];
1099  break;
1100  }
1101  }
1102  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
1103  // Default if no sample formats found
1104  c->sample_fmt = AV_SAMPLE_FMT_S16;
1105  }
1106 
1107  // some formats want stream headers to be separate
1108  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1109 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1110  // FFmpeg 3.0+
1111  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1112 #else
1113  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1114 #endif
1115 
1116  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1117 
1118  int nb_channels;
1119  const char* nb_channels_label;
1120  const char* channel_layout_label;
1121 
1122 #if HAVE_CH_LAYOUT
1123  nb_channels = c->ch_layout.nb_channels;
1124  channel_layout = c->ch_layout.u.mask;
1125  nb_channels_label = "c->ch_layout.nb_channels";
1126  channel_layout_label = "c->ch_layout.u.mask";
1127 #else
1128  nb_channels = c->channels;
1129  nb_channels_label = "c->channels";
1130  channel_layout_label = "c->channel_layout";
1131 #endif
1132 
1133  ZmqLogger::Instance()->AppendDebugMethod(
1134  "FFmpegWriter::add_audio_stream",
1135  "c->codec_id", c->codec_id,
1136  "c->bit_rate", c->bit_rate,
1137  nb_channels_label, nb_channels,
1138  "c->sample_fmt", c->sample_fmt,
1139  channel_layout_label, channel_layout,
1140  "c->sample_rate", c->sample_rate);
1141 
1142  return st;
1143 }
1144 
1145 // Add a video output stream
1146 AVStream *FFmpegWriter::add_video_stream() {
1147  // Find the video codec
1148  const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1149  if (codec == NULL)
1150  throw InvalidCodec("A valid video codec could not be found for this file.", path);
1151 
1152  // Free any previous memory allocations
1153  if (video_codec_ctx != nullptr) {
1154  AV_FREE_CONTEXT(video_codec_ctx);
1155  }
1156 
1157  // Create a new video stream
1158  AVStream* st = avformat_new_stream(oc, codec);
1159  if (!st)
1160  throw OutOfMemory("Could not allocate memory for the video stream.", path);
1161 
1162  // Allocate a new codec context for the stream
1163  ALLOC_CODEC_CTX(video_codec_ctx, codec, st)
1164 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1165  st->codecpar->codec_id = codec->id;
1166 #endif
1167 
1168  AVCodecContext* c = video_codec_ctx;
1169 
1170  c->codec_id = codec->id;
1171  c->codec_type = AVMEDIA_TYPE_VIDEO;
1172 
1173  // Set sample aspect ratio
1174  c->sample_aspect_ratio.num = info.pixel_ratio.num;
1175  c->sample_aspect_ratio.den = info.pixel_ratio.den;
1176 
1177  /* Init video encoder options */
1178  if (info.video_bit_rate >= 1000
1179 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1180  && c->codec_id != AV_CODEC_ID_AV1
1181 #endif
1182  ) {
1183  c->bit_rate = info.video_bit_rate;
1184  if (info.video_bit_rate >= 1500000) {
1185  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1186  c->qmin = 2;
1187  c->qmax = 30;
1188  }
1189  }
1190  // Here should be the setting for low fixed bitrate
1191  // Defaults are used because mpeg2 otherwise had problems
1192  } else {
1193  // Check if codec supports crf or qp
1194  switch (c->codec_id) {
1195 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1196  // FFmpeg 4.0+
1197  case AV_CODEC_ID_AV1 :
1198  // TODO: Set `crf` or `qp` according to bitrate, as bitrate is not supported by these encoders yet.
1199  if (info.video_bit_rate >= 1000) {
1200  c->bit_rate = 0;
1201  if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1202  int calculated_quality = 35;
1203  if (info.video_bit_rate < 500000) calculated_quality = 50;
1204  if (info.video_bit_rate > 5000000) calculated_quality = 10;
1205  av_opt_set_int(c->priv_data, "crf", calculated_quality, 0);
1206  info.video_bit_rate = calculated_quality;
1207  } else {
1208  int calculated_quality = 50;
1209  if (info.video_bit_rate < 500000) calculated_quality = 60;
1210  if (info.video_bit_rate > 5000000) calculated_quality = 15;
1211  av_opt_set_int(c->priv_data, "qp", calculated_quality, 0);
1212  info.video_bit_rate = calculated_quality;
1213  } // medium
1214  }
1215  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
1216  av_opt_set_int(c->priv_data, "preset", 6, 0);
1217  av_opt_set_int(c->priv_data, "forced-idr",1,0);
1218  }
1219  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
1220  av_opt_set_int(c->priv_data, "speed", 7, 0);
1221  av_opt_set_int(c->priv_data, "tile-rows", 2, 0);
1222  av_opt_set_int(c->priv_data, "tile-columns", 4, 0);
1223  }
1224  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1225  // Set number of tiles to a fixed value
1226  // TODO: Allow user to chose their own number of tiles
1227  av_opt_set_int(c->priv_data, "tile-rows", 1, 0); // log2 of number of rows
1228  av_opt_set_int(c->priv_data, "tile-columns", 2, 0); // log2 of number of columns
1229  av_opt_set_int(c->priv_data, "row-mt", 1, 0); // use multiple cores
1230  av_opt_set_int(c->priv_data, "cpu-used", 3, 0); // default is 1, usable is 4
1231  }
1232  //break;
1233 #endif
1234  case AV_CODEC_ID_VP9 :
1235  case AV_CODEC_ID_HEVC :
1236  case AV_CODEC_ID_VP8 :
1237  case AV_CODEC_ID_H264 :
1238  if (info.video_bit_rate < 40) {
1239  c->qmin = 0;
1240  c->qmax = 63;
1241  } else {
1242  c->qmin = info.video_bit_rate - 5;
1243  c->qmax = 63;
1244  }
1245  break;
1246  default:
1247  // Here should be the setting for codecs that don't support crf
1248  // For now defaults are used
1249  break;
1250  }
1251  }
1252 
1253  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
1254  //invalid bitrate errors and rc buffer underflow errors, etc...
1255  //c->rc_min_rate = info.video_bit_rate;
1256  //c->rc_max_rate = info.video_bit_rate;
1257  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
1258  //if ( !c->rc_initial_buffer_occupancy )
1259  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
1260 
1261  /* resolution must be a multiple of two */
1262  // TODO: require /2 height and width
1263  c->width = info.width;
1264  c->height = info.height;
1265 
1266  /* time base: this is the fundamental unit of time (in seconds) in terms
1267  of which frame timestamps are represented. for fixed-fps content,
1268  timebase should be 1/framerate and timestamp increments should be
1269  identically 1. */
1270  c->time_base.num = info.video_timebase.num;
1271  c->time_base.den = info.video_timebase.den;
1272 // AVCodecContext->framerate was added in FFmpeg 2.6
1273 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 26, 0)
1274  c->framerate = av_inv_q(c->time_base);
1275 #endif
1276  st->avg_frame_rate = av_inv_q(c->time_base);
1277  st->time_base.num = info.video_timebase.num;
1278  st->time_base.den = info.video_timebase.den;
1279 
1280  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
1281  c->max_b_frames = 10;
1282  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
1283  /* just for testing, we also add B frames */
1284  c->max_b_frames = 2;
1285  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
1286  /* Needed to avoid using macroblocks in which some coeffs overflow.
1287  This does not happen with normal video, it just happens here as
1288  the motion of the chroma plane does not match the luma plane. */
1289  c->mb_decision = 2;
1290  // some formats want stream headers to be separate
1291  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1292 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1293  // FFmpeg 3.0+
1294  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1295 #else
1296  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1297 #endif
1298 
1299  // Find all supported pixel formats for this codec
1300  const PixelFormat *supported_pixel_formats = codec->pix_fmts;
1301  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
1302  // Assign the 1st valid pixel format (if one is missing)
1303  if (c->pix_fmt == PIX_FMT_NONE)
1304  c->pix_fmt = *supported_pixel_formats;
1305  ++supported_pixel_formats;
1306  }
1307 
1308  // Codec doesn't have any pix formats?
1309  if (c->pix_fmt == PIX_FMT_NONE) {
1310  if (oc->oformat->video_codec == AV_CODEC_ID_RAWVIDEO) {
1311  // Raw video should use RGB24
1312  c->pix_fmt = PIX_FMT_RGB24;
1313 
1314 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1315  // FFmpeg < 4.0
1316  if (strcmp(oc->oformat->name, "gif") != 0)
1317  // If not GIF format, skip the encoding process
1318  // Set raw picture flag (so we don't encode this video)
1319  oc->oformat->flags |= AVFMT_RAWPICTURE;
1320 #endif
1321  } else {
1322  // Set the default codec
1323  c->pix_fmt = PIX_FMT_YUV420P;
1324  }
1325  }
1326 
1327  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1328  ZmqLogger::Instance()->AppendDebugMethod(
1329  "FFmpegWriter::add_video_stream ("
1330  + (std::string)oc->oformat->name + " : "
1331  + (std::string)av_get_pix_fmt_name(c->pix_fmt) + ")",
1332  "c->codec_id", c->codec_id,
1333  "c->bit_rate", c->bit_rate,
1334  "c->pix_fmt", c->pix_fmt,
1335  "oc->oformat->flags", oc->oformat->flags);
1336  return st;
1337 }
1338 
1339 // open audio codec
1340 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) {
1341  const AVCodec *codec;
1342  AV_GET_CODEC_FROM_STREAM(st, audio_codec_ctx)
1343 
1344  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
1345  audio_codec_ctx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
1346 
1347  // Find the audio encoder
1348  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1349  if (!codec)
1350  codec = avcodec_find_encoder(audio_codec_ctx->codec_id);
1351  if (!codec)
1352  throw InvalidCodec("Could not find codec", path);
1353 
1354  // Init options
1355  AVDictionary *opts = NULL;
1356  av_dict_set(&opts, "strict", "experimental", 0);
1357 
1358  // Open the codec
1359  if (avcodec_open2(audio_codec_ctx, codec, &opts) < 0)
1360  throw InvalidCodec("Could not open audio codec", path);
1361  AV_COPY_PARAMS_FROM_CONTEXT(st, audio_codec_ctx);
1362 
1363  // Free options
1364  av_dict_free(&opts);
1365 
1366  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
1367  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
1368  if (audio_codec_ctx->frame_size <= 1) {
1369  // No frame size found... so calculate
1370  audio_input_frame_size = 50000 / info.channels;
1371 
1372  int s = AV_FIND_DECODER_CODEC_ID(st);
1373  switch (s) {
1374  case AV_CODEC_ID_PCM_S16LE:
1375  case AV_CODEC_ID_PCM_S16BE:
1376  case AV_CODEC_ID_PCM_U16LE:
1377  case AV_CODEC_ID_PCM_U16BE:
1378  audio_input_frame_size >>= 1;
1379  break;
1380  default:
1381  break;
1382  }
1383  } else {
1384  // Set frame size based on the codec
1385  audio_input_frame_size = audio_codec_ctx->frame_size;
1386  }
1387 
1388  // Set the initial frame size (since it might change during resampling)
1389  initial_audio_input_frame_size = audio_input_frame_size;
1390 
1391  // Allocate array for samples
1392  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1393 
1394  // Set audio output buffer (used to store the encoded audio)
1395  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1396  audio_outbuf = new uint8_t[audio_outbuf_size];
1397 
1398  // Set audio packet encoding buffer
1399  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1400  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1401 
1402  // Add audio metadata (if any)
1403  for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1404  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1405  }
1406 
1407  ZmqLogger::Instance()->AppendDebugMethod(
1408  "FFmpegWriter::open_audio",
1409  "audio_codec_ctx->thread_count", audio_codec_ctx->thread_count,
1410  "audio_input_frame_size", audio_input_frame_size,
1411  "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE);
1412 }
1413 
1414 // open video codec
1415 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
1416  const AVCodec *codec;
1417  AV_GET_CODEC_FROM_STREAM(st, video_codec_ctx)
1418 
1419  // Set number of threads equal to number of processors (not to exceed 16, FFmpeg doesn't recommend more than 16)
1420  video_codec_ctx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
1421 
1422 #if USE_HW_ACCEL
1423  if (hw_en_on && hw_en_supported) {
1424  //char *dev_hw = NULL;
1425  char adapter[256];
1426  char *adapter_ptr = NULL;
1427  int adapter_num;
1428  // Use the hw device given in the environment variable HW_EN_DEVICE_SET or the default if not set
1429  adapter_num = openshot::Settings::Instance()->HW_EN_DEVICE_SET;
1430  std::clog << "Encoding Device Nr: " << adapter_num << "\n";
1431  if (adapter_num < 3 && adapter_num >=0) {
1432 #if defined(__linux__)
1433  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
1434  // Maybe 127 is better because the first card would be 1?!
1435  adapter_ptr = adapter;
1436 #elif defined(_WIN32) || defined(__APPLE__)
1437  adapter_ptr = NULL;
1438 #endif
1439  }
1440  else {
1441  adapter_ptr = NULL; // Just to be sure
1442  }
1443 // Check if it is there and writable
1444 #if defined(__linux__)
1445  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
1446 #elif defined(_WIN32) || defined(__APPLE__)
1447  if( adapter_ptr != NULL ) {
1448 #endif
1449  ZmqLogger::Instance()->AppendDebugMethod(
1450  "Encode Device present using device",
1451  "adapter", adapter_num);
1452  }
1453  else {
1454  adapter_ptr = NULL; // use default
1455  ZmqLogger::Instance()->AppendDebugMethod(
1456  "Encode Device not present, using default");
1457  }
1458  if (av_hwdevice_ctx_create(&hw_device_ctx,
1459  hw_en_av_device_type, adapter_ptr, NULL, 0) < 0)
1460  {
1461  ZmqLogger::Instance()->AppendDebugMethod(
1462  "FFmpegWriter::open_video ERROR creating hwdevice, Codec name:",
1463  info.vcodec.c_str(), -1);
1464  throw InvalidCodec("Could not create hwdevice", path);
1465  }
1466  }
1467 #endif // USE_HW_ACCEL
1468 
1469  /* find the video encoder */
1470  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1471  if (!codec)
1472  codec = avcodec_find_encoder(AV_FIND_DECODER_CODEC_ID(st));
1473  if (!codec)
1474  throw InvalidCodec("Could not find codec", path);
1475 
1476  /* Legacy behavior: force max_b_frames to 0 for many codecs.
1477  * This can be disabled via SetOption(VIDEO_STREAM, "allow_b_frames", "1"). */
1478  if (!allow_b_frames && video_codec_ctx->max_b_frames &&
1479  video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 &&
1480  video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
1481  video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1482  video_codec_ctx->max_b_frames = 0;
1483 
1484  // Init options
1485  AVDictionary *opts = NULL;
1486  av_dict_set(&opts, "strict", "experimental", 0);
1487 
1488 #if USE_HW_ACCEL
1489  if (hw_en_on && hw_en_supported) {
1490  video_codec_ctx->pix_fmt = hw_en_av_pix_fmt;
1491 
1492  // for the list of possible options, see the list of codec-specific options:
1493  // e.g. ffmpeg -h encoder=h264_vaapi or ffmpeg -h encoder=hevc_vaapi
1494  // and "man ffmpeg-codecs"
1495 
1496  // For VAAPI, it is safer to explicitly set rc_mode instead of relying on auto-selection
1497  // which is ffmpeg version-specific.
1498  if (hw_en_av_pix_fmt == AV_PIX_FMT_VAAPI) {
1499  int64_t qp;
1500  if (av_opt_get_int(video_codec_ctx->priv_data, "qp", 0, &qp) != 0 || qp == 0) {
1501  // unless "qp" was set for CQP, switch to VBR RC mode
1502  av_opt_set(video_codec_ctx->priv_data, "rc_mode", "VBR", 0);
1503 
1504  // In the current state (ffmpeg-4.2-4 libva-mesa-driver-19.1.5-1) to use VBR,
1505  // one has to specify both bit_rate and maxrate, otherwise a small low quality file is generated on Intel iGPU).
1506  video_codec_ctx->rc_max_rate = video_codec_ctx->bit_rate;
1507  }
1508  }
1509 
1510  switch (video_codec_ctx->codec_id) {
1511  case AV_CODEC_ID_H264:
1512  video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames
1513  video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE;
1514  av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
1515  av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
1516  av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
1517  break;
1518  case AV_CODEC_ID_HEVC:
1519  // tested to work with defaults
1520  break;
1521  case AV_CODEC_ID_VP9:
1522  // tested to work with defaults
1523  break;
1524  default:
1525  ZmqLogger::Instance()->AppendDebugMethod(
1526  "No codec-specific options defined for this codec. HW encoding may fail",
1527  "codec_id", video_codec_ctx->codec_id);
1528  break;
1529  }
1530 
1531  // set hw_frames_ctx for encoder's AVCodecContext
1532  int err;
1533  if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0)
1534  {
1535  ZmqLogger::Instance()->AppendDebugMethod(
1536  "FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context",
1537  "width", info.width,
1538  "height", info.height,
1539  av_err2string(err), -1);
1540  }
1541  }
1542 #endif // USE_HW_ACCEL
1543 
1544 // Set libx265 hvc1 tag (for Apple playback compatibility).
1545 #if USE_HW_ACCEL
1546  if (!(hw_en_on && hw_en_supported) && video_codec_ctx->codec_id == AV_CODEC_ID_HEVC) {
1547  video_codec_ctx->codec_tag = MKTAG('h', 'v', 'c', '1');
1548  }
1549 #else
1550  if (video_codec_ctx->codec_id == AV_CODEC_ID_HEVC) {
1551  video_codec_ctx->codec_tag = MKTAG('h', 'v', 'c', '1');
1552  }
1553 #endif
1554 
1555  /* open the codec */
1556  if (avcodec_open2(video_codec_ctx, codec, &opts) < 0)
1557  throw InvalidCodec("Could not open video codec", path);
1558  AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx);
1559 
1560  // Free options
1561  av_dict_free(&opts);
1562 
1563  // Add video metadata (if any)
1564  for (auto iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1565  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1566  }
1567 
1568  ZmqLogger::Instance()->AppendDebugMethod(
1569  "FFmpegWriter::open_video",
1570  "video_codec_ctx->thread_count", video_codec_ctx->thread_count);
1571 
1572 }
1573 
1574 // write all queued frames' audio to the video file
1575 void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame) {
1576  if (!frame && !is_final)
1577  return;
1578 
1579  // Init audio buffers / variables
1580  int total_frame_samples = 0;
1581  int frame_position = 0;
1582  int channels_in_frame = 0;
1583  int sample_rate_in_frame = 0;
1584  int samples_in_frame = 0;
1585  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1586 
1587  // Create a new array (to hold all S16 audio samples, for the current queued frames
1588  unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1589  int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
1590  int16_t *all_resampled_samples = NULL;
1591  int16_t *final_samples_planar = NULL;
1592  int16_t *final_samples = NULL;
1593 
1594  // Get audio sample array
1595  float *frame_samples_float = NULL;
1596 
1597  // Get the audio details from this frame
1598  if (frame) {
1599  sample_rate_in_frame = frame->SampleRate();
1600  samples_in_frame = frame->GetAudioSamplesCount();
1601  channels_in_frame = frame->GetAudioChannelsCount();
1602  channel_layout_in_frame = frame->ChannelsLayout();
1603 
1604  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1605  frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
1606  }
1607 
1608  // Calculate total samples
1609  total_frame_samples = samples_in_frame * channels_in_frame;
1610 
1611  // Translate audio sample values back to 16 bit integers with saturation
1612  const int16_t max16 = 32767;
1613  const int16_t min16 = -32768;
1614  for (int s = 0; s < total_frame_samples; s++, frame_position++) {
1615  float valF = frame_samples_float[s] * (1 << 15);
1616  int16_t conv;
1617  if (valF > max16) {
1618  conv = max16;
1619  } else if (valF < min16) {
1620  conv = min16;
1621  } else {
1622  conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
1623  }
1624 
1625  // Copy into buffer
1626  all_queued_samples[frame_position] = conv;
1627  }
1628 
1629  // Deallocate float array
1630  delete[] frame_samples_float;
1631 
1632 
1633  // Update total samples (since we've combined all queued frames)
1634  total_frame_samples = frame_position;
1635  int remaining_frame_samples = total_frame_samples;
1636  int samples_position = 0;
1637 
1638 
1639  ZmqLogger::Instance()->AppendDebugMethod(
1640  "FFmpegWriter::write_audio_packets",
1641  "is_final", is_final,
1642  "total_frame_samples", total_frame_samples,
1643  "channel_layout_in_frame", channel_layout_in_frame,
1644  "channels_in_frame", channels_in_frame,
1645  "samples_in_frame", samples_in_frame,
1646  "LAYOUT_MONO", LAYOUT_MONO);
1647 
1648  // Keep track of the original sample format
1649  AVSampleFormat output_sample_fmt = audio_codec_ctx->sample_fmt;
1650 
1651  AVFrame *audio_frame = NULL;
1652  if (!is_final) {
1653  // Create input frame (and allocate arrays)
1654  audio_frame = AV_ALLOCATE_FRAME();
1655  AV_RESET_FRAME(audio_frame);
1656  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1657 
1658  // Fill input frame with sample data
1659  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples, all_queued_samples_size, 0);
1660  if (error_code < 0) {
1661  ZmqLogger::Instance()->AppendDebugMethod(
1662  "FFmpegWriter::write_audio_packets ERROR ["
1663  + av_err2string(error_code) + "]",
1664  "error_code", error_code);
1665  }
1666 
1667  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1668  switch (audio_codec_ctx->sample_fmt) {
1669  case AV_SAMPLE_FMT_FLTP: {
1670  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1671  break;
1672  }
1673  case AV_SAMPLE_FMT_S32P: {
1674  output_sample_fmt = AV_SAMPLE_FMT_S32;
1675  break;
1676  }
1677  case AV_SAMPLE_FMT_S16P: {
1678  output_sample_fmt = AV_SAMPLE_FMT_S16;
1679  break;
1680  }
1681  case AV_SAMPLE_FMT_U8P: {
1682  output_sample_fmt = AV_SAMPLE_FMT_U8;
1683  break;
1684  }
1685  default: {
1686  // This is only here to silence unused-enum warnings
1687  break;
1688  }
1689  }
1690 
1691  // Update total samples & input frame size (due to bigger or smaller data types)
1692  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1693  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1694 
1695  // Create output frame (and allocate arrays)
1696  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1697  AV_RESET_FRAME(audio_converted);
1698  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1699  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1700 
1701  ZmqLogger::Instance()->AppendDebugMethod(
1702  "FFmpegWriter::write_audio_packets (1st resampling)",
1703  "in_sample_fmt", AV_SAMPLE_FMT_S16,
1704  "out_sample_fmt", output_sample_fmt,
1705  "in_sample_rate", sample_rate_in_frame,
1706  "out_sample_rate", info.sample_rate,
1707  "in_channels", channels_in_frame,
1708  "out_channels", info.channels);
1709 
1710  // setup resample context
1711  if (!avr) {
1712  avr = SWR_ALLOC();
1713 #if HAVE_CH_LAYOUT
1714  AVChannelLayout in_chlayout = ffmpeg_default_channel_layout(channels_in_frame);
1715  AVChannelLayout out_chlayout = ffmpeg_default_channel_layout(info.channels);
1716  if (channel_layout_in_frame > 0) {
1717  av_channel_layout_from_mask(&in_chlayout, channel_layout_in_frame);
1718  }
1719  if (info.channel_layout > 0) {
1720  av_channel_layout_from_mask(&out_chlayout, info.channel_layout);
1721  }
1722  av_opt_set_chlayout(avr, "in_chlayout", &in_chlayout, 0);
1723  av_opt_set_chlayout(avr, "out_chlayout", &out_chlayout, 0);
1724 #else
1725  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1726  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1727  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1728  av_opt_set_int(avr, "out_channels", info.channels, 0);
1729 #endif
1730  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1731  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1732  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1733  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1734  SWR_INIT(avr);
1735  }
1736  // Convert audio samples
1737  int nb_samples = SWR_CONVERT(
1738  avr, // audio resample context
1739  audio_converted->data, // output data pointers
1740  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1741  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1742  audio_frame->data, // input data pointers
1743  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1744  audio_frame->nb_samples // number of input samples to convert
1745  );
1746 
1747  // Set remaining samples
1748  remaining_frame_samples = total_frame_samples;
1749 
1750  // Create a new array (to hold all resampled S16 audio samples)
1751  all_resampled_samples = (int16_t *) av_malloc(
1752  sizeof(int16_t) * nb_samples * info.channels
1753  * (av_get_bytes_per_sample(output_sample_fmt) /
1754  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1755  );
1756 
1757  // Copy audio samples over original samples
1758  memcpy(all_resampled_samples, audio_converted->data[0],
1759  static_cast<size_t>(nb_samples)
1760  * info.channels
1761  * av_get_bytes_per_sample(output_sample_fmt));
1762 
1763  // Remove converted audio
1764  av_freep(&(audio_frame->data[0]));
1765  AV_FREE_FRAME(&audio_frame);
1766  av_freep(&audio_converted->data[0]);
1767  AV_FREE_FRAME(&audio_converted);
1768  all_queued_samples = NULL; // this array cleared with above call
1769 
1770  ZmqLogger::Instance()->AppendDebugMethod(
1771  "FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)",
1772  "nb_samples", nb_samples,
1773  "remaining_frame_samples", remaining_frame_samples);
1774  }
1775 
1776  // Loop until no more samples
1777  while (remaining_frame_samples > 0 || is_final) {
1778  // Get remaining samples needed for this packet
1779  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1780 
1781  // Determine how many samples we need
1782  int diff = 0;
1783  if (remaining_frame_samples >= remaining_packet_samples) {
1784  diff = remaining_packet_samples;
1785  } else {
1786  diff = remaining_frame_samples;
1787  }
1788 
1789  // Copy frame samples into the packet samples array
1790  if (!is_final)
1791  //TODO: Make this more sane
1792  memcpy(
1793  samples + (audio_input_position
1794  * (av_get_bytes_per_sample(output_sample_fmt) /
1795  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1796  ),
1797  all_resampled_samples + samples_position,
1798  static_cast<size_t>(diff)
1799  * av_get_bytes_per_sample(output_sample_fmt)
1800  );
1801 
1802  // Increment counters
1803  audio_input_position += diff;
1804  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1805  remaining_frame_samples -= diff;
1806 
1807  // Do we have enough samples to proceed?
1808  if (audio_input_position < (audio_input_frame_size * info.channels) && !is_final)
1809  // Not enough samples to encode... so wait until the next frame
1810  break;
1811 
1812  // Convert to planar (if needed by audio codec)
1813  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1814  AV_RESET_FRAME(frame_final);
1815  if (av_sample_fmt_is_planar(audio_codec_ctx->sample_fmt)) {
1816  ZmqLogger::Instance()->AppendDebugMethod(
1817  "FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)",
1818  "in_sample_fmt", output_sample_fmt,
1819  "out_sample_fmt", audio_codec_ctx->sample_fmt,
1820  "in_sample_rate", info.sample_rate,
1821  "out_sample_rate", info.sample_rate,
1822  "in_channels", info.channels,
1823  "out_channels", info.channels
1824  );
1825 
1826  // setup resample context
1827  if (!avr_planar) {
1828  avr_planar = SWR_ALLOC();
1829 #if HAVE_CH_LAYOUT
1830  AVChannelLayout layout = ffmpeg_default_channel_layout(info.channels);
1831  if (info.channel_layout > 0) {
1832  av_channel_layout_from_mask(&layout, info.channel_layout);
1833  }
1834  av_opt_set_chlayout(avr_planar, "in_chlayout", &layout, 0);
1835  av_opt_set_chlayout(avr_planar, "out_chlayout", &layout, 0);
1836 #else
1837  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1838  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1839  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1840  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1841 #endif
1842  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1843  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec_ctx->sample_fmt, 0); // planar not allowed here
1844  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1845  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1846  SWR_INIT(avr_planar);
1847  }
1848 
1849  // Create input frame (and allocate arrays)
1850  audio_frame = AV_ALLOCATE_FRAME();
1851  AV_RESET_FRAME(audio_frame);
1852  audio_frame->nb_samples = audio_input_position / info.channels;
1853 
1854  // Create a new array
1855  final_samples_planar = (int16_t *) av_malloc(
1856  sizeof(int16_t) * audio_frame->nb_samples * info.channels
1857  * (av_get_bytes_per_sample(output_sample_fmt) /
1858  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1859  );
1860 
1861  // Copy audio into buffer for frame
1862  memcpy(final_samples_planar, samples,
1863  static_cast<size_t>(audio_frame->nb_samples)
1864  * info.channels
1865  * av_get_bytes_per_sample(output_sample_fmt));
1866 
1867  // Fill input frame with sample data
1868  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt,
1869  (uint8_t *) final_samples_planar, audio_encoder_buffer_size, 0);
1870 
1871  // Create output frame (and allocate arrays)
1872  frame_final->nb_samples = audio_input_frame_size;
1873 #if HAVE_CH_LAYOUT
1874  av_channel_layout_from_mask(&frame_final->ch_layout, info.channel_layout);
1875 #else
1876  frame_final->channels = info.channels;
1877  frame_final->channel_layout = info.channel_layout;
1878 #endif
1879  frame_final->format = audio_codec_ctx->sample_fmt;
1880  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels,
1881  frame_final->nb_samples, audio_codec_ctx->sample_fmt, 0);
1882 
1883  // Convert audio samples
1884  int nb_samples = SWR_CONVERT(
1885  avr_planar, // audio resample context
1886  frame_final->data, // output data pointers
1887  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1888  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1889  audio_frame->data, // input data pointers
1890  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1891  audio_frame->nb_samples // number of input samples to convert
1892  );
1893 
1894  // Copy audio samples over original samples
1895  const auto copy_length = static_cast<size_t>(nb_samples)
1896  * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt)
1897  * info.channels;
1898 
1899  if (nb_samples > 0)
1900  memcpy(samples, frame_final->data[0], copy_length);
1901 
1902  // deallocate AVFrame
1903  av_freep(&(audio_frame->data[0]));
1904  AV_FREE_FRAME(&audio_frame);
1905  all_queued_samples = NULL; // this array cleared with above call
1906 
1907  ZmqLogger::Instance()->AppendDebugMethod(
1908  "FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)",
1909  "nb_samples", nb_samples);
1910 
1911  } else {
1912  // Create a new array
1913  const auto buf_size = static_cast<size_t>(audio_input_position)
1914  * (av_get_bytes_per_sample(audio_codec_ctx->sample_fmt) /
1915  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)
1916  );
1917  final_samples = reinterpret_cast<int16_t*>(
1918  av_malloc(sizeof(int16_t) * buf_size));
1919 
1920  // Copy audio into buffer for frame
1921  memcpy(final_samples, samples,
1922  audio_input_position * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt));
1923 
1924  // Init the nb_samples property
1925  frame_final->nb_samples = audio_input_frame_size;
1926 
1927  // Fill the final_frame AVFrame with audio (non planar)
1928 #if HAVE_CH_LAYOUT
1929  int nb_channels = audio_codec_ctx->ch_layout.nb_channels;
1930 #else
1931  int nb_channels = audio_codec_ctx->channels;
1932 #endif
1933  avcodec_fill_audio_frame(frame_final, nb_channels,
1934  audio_codec_ctx->sample_fmt, (uint8_t *) final_samples,
1935  audio_encoder_buffer_size, 0);
1936  }
1937 
1938  // Set the AVFrame's PTS
1939  frame_final->pts = audio_timestamp;
1940 
1941  // Init the packet
1942 #if IS_FFMPEG_3_2
1943  AVPacket* pkt = av_packet_alloc();
1944 #else
1945  AVPacket* pkt;
1946  av_init_packet(pkt);
1947 #endif
1948  pkt->data = audio_encoder_buffer;
1949  pkt->size = audio_encoder_buffer_size;
1950 
1951  // Set the packet's PTS prior to encoding
1952  pkt->pts = pkt->dts = audio_timestamp;
1953 
1954  /* encode the audio samples */
1955  int got_packet_ptr = 0;
1956 
1957 #if IS_FFMPEG_3_2
1958  // Encode audio (latest version of FFmpeg)
1959  int error_code;
1960  int ret = 0;
1961  int frame_finished = 0;
1962  error_code = ret = avcodec_send_frame(audio_codec_ctx, frame_final);
1963  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1964  avcodec_send_frame(audio_codec_ctx, NULL);
1965  }
1966  else {
1967  if (ret >= 0)
1968  pkt->size = 0;
1969  ret = avcodec_receive_packet(audio_codec_ctx, pkt);
1970  if (ret >= 0)
1971  frame_finished = 1;
1972  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1973  avcodec_flush_buffers(audio_codec_ctx);
1974  ret = 0;
1975  }
1976  if (ret >= 0) {
1977  ret = frame_finished;
1978  }
1979  }
1980  if (!pkt->data && !frame_finished)
1981  {
1982  ret = -1;
1983  }
1984  got_packet_ptr = ret;
1985 #else
1986  // Encode audio (older versions of FFmpeg)
1987  int error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, frame_final, &got_packet_ptr);
1988 #endif
1989  /* if zero size, it means the image was buffered */
1990  if (error_code == 0 && got_packet_ptr) {
1991 
1992  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1993  // but it fixes lots of PTS related issues when I do this.
1994  pkt->pts = pkt->dts = audio_timestamp;
1995 
1996  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1997  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
1998 
1999  // set stream
2000  pkt->stream_index = audio_st->index;
2001  pkt->flags |= AV_PKT_FLAG_KEY;
2002 
2003  /* write the compressed frame in the media file */
2004  error_code = av_interleaved_write_frame(oc, pkt);
2005  }
2006 
2007  if (error_code < 0) {
2008  ZmqLogger::Instance()->AppendDebugMethod(
2009  "FFmpegWriter::write_audio_packets ERROR ["
2010  + av_err2string(error_code) + "]",
2011  "error_code", error_code);
2012  }
2013 
2014  // Increment PTS (no pkt.duration, so calculate with maths)
2015  audio_timestamp += FFMIN(audio_input_frame_size, audio_input_position);
2016 
2017  // deallocate AVFrame
2018  av_freep(&(frame_final->data[0]));
2019  AV_FREE_FRAME(&frame_final);
2020 
2021  // deallocate memory for packet
2022  AV_FREE_PACKET(pkt);
2023 
2024  // Reset position
2025  audio_input_position = 0;
2026  is_final = false;
2027  }
2028 
2029  // Delete arrays (if needed)
2030  if (all_resampled_samples) {
2031  av_freep(&all_resampled_samples);
2032  all_resampled_samples = NULL;
2033  }
2034  if (all_queued_samples) {
2035  av_freep(&all_queued_samples);
2036  all_queued_samples = NULL;
2037  }
2038 }
2039 
2040 // Allocate an AVFrame object
2041 AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) {
2042  // Create an RGB AVFrame
2043  AVFrame *new_av_frame = NULL;
2044 
2045  // Allocate an AVFrame structure
2046  new_av_frame = AV_ALLOCATE_FRAME();
2047  if (new_av_frame == NULL)
2048  throw OutOfMemory("Could not allocate AVFrame", path);
2049 
2050  // Determine required buffer size and allocate buffer
2051  *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height);
2052 
2053  // Create buffer (if not provided)
2054  if (!new_buffer) {
2055  // New Buffer
2056  new_buffer = (uint8_t *) av_malloc(*buffer_size * sizeof(uint8_t));
2057  // Attach buffer to AVFrame
2058  AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height);
2059  new_av_frame->width = width;
2060  new_av_frame->height = height;
2061  new_av_frame->format = pix_fmt;
2062  }
2063 
2064  // return AVFrame
2065  return new_av_frame;
2066 }
2067 
2068 // process video frame
2069 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame) {
2070  // Source dimensions (RGBA)
2071  int src_w = frame->GetWidth();
2072  int src_h = frame->GetHeight();
2073 
2074  // Skip empty frames (1×1)
2075  if (src_w == 1 && src_h == 1)
2076  return;
2077 
2078  // Point persistent_src_frame->data to RGBA pixels
2079  const uchar* pixels = frame->GetPixels();
2080  if (!persistent_src_frame) {
2081  persistent_src_frame = av_frame_alloc();
2082  if (!persistent_src_frame)
2083  throw OutOfMemory("Could not allocate persistent_src_frame", path);
2084  persistent_src_frame->format = AV_PIX_FMT_RGBA;
2085  persistent_src_frame->width = src_w;
2086  persistent_src_frame->height = src_h;
2087  persistent_src_frame->linesize[0] = src_w * 4;
2088  }
2089  persistent_src_frame->data[0] = const_cast<uint8_t*>(
2090  reinterpret_cast<const uint8_t*>(pixels)
2091  );
2092 
2093  // Prepare persistent_dst_frame + buffer on first use
2094  if (!persistent_dst_frame) {
2095  persistent_dst_frame = av_frame_alloc();
2096  if (!persistent_dst_frame)
2097  throw OutOfMemory("Could not allocate persistent_dst_frame", path);
2098 
2099  // Decide destination pixel format: NV12 if HW accel is on, else encoder’s pix_fmt
2100  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2101 #if USE_HW_ACCEL
2102  if (hw_en_on && hw_en_supported) {
2103  dst_fmt = AV_PIX_FMT_NV12;
2104  }
2105 #endif
2106  persistent_dst_frame->format = dst_fmt;
2107  persistent_dst_frame->width = info.width;
2108  persistent_dst_frame->height = info.height;
2109 
2110  persistent_dst_size = av_image_get_buffer_size(
2111  dst_fmt, info.width, info.height, 1
2112  );
2113  if (persistent_dst_size < 0)
2114  throw ErrorEncodingVideo("Invalid destination image size", -1);
2115 
2116  persistent_dst_buffer = static_cast<uint8_t*>(
2117  av_malloc(persistent_dst_size)
2118  );
2119  if (!persistent_dst_buffer)
2120  throw OutOfMemory("Could not allocate persistent_dst_buffer", path);
2121 
2122  av_image_fill_arrays(
2123  persistent_dst_frame->data,
2124  persistent_dst_frame->linesize,
2125  persistent_dst_buffer,
2126  dst_fmt,
2127  info.width,
2128  info.height,
2129  1
2130  );
2131  }
2132 
2133  // Initialize SwsContext (RGBA → dst_fmt) on first use
2134  if (!img_convert_ctx) {
2135  int flags = SWS_FAST_BILINEAR;
2136  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
2137  flags = SWS_BICUBIC;
2138  }
2139  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2140 #if USE_HW_ACCEL
2141  if (hw_en_on && hw_en_supported) {
2142  dst_fmt = AV_PIX_FMT_NV12;
2143  }
2144 #endif
2145  img_convert_ctx = sws_getContext(
2146  src_w, src_h, AV_PIX_FMT_RGBA,
2147  info.width, info.height, dst_fmt,
2148  flags, NULL, NULL, NULL
2149  );
2150  if (!img_convert_ctx)
2151  throw ErrorEncodingVideo("Could not initialize sws context", -1);
2152  }
2153 
2154  // Scale RGBA → dst_fmt into persistent_dst_buffer
2155  sws_scale(
2156  img_convert_ctx,
2157  persistent_src_frame->data,
2158  persistent_src_frame->linesize,
2159  0, src_h,
2160  persistent_dst_frame->data,
2161  persistent_dst_frame->linesize
2162  );
2163 
2164  // Allocate a new AVFrame + buffer, then copy scaled data into it
2165  int bytes_final = 0;
2166  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2167 #if USE_HW_ACCEL
2168  if (hw_en_on && hw_en_supported) {
2169  dst_fmt = AV_PIX_FMT_NV12;
2170  }
2171 #endif
2172 
2173  AVFrame* new_frame = allocate_avframe(
2174  dst_fmt,
2175  info.width,
2176  info.height,
2177  &bytes_final,
2178  nullptr
2179  );
2180  if (!new_frame)
2181  throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path);
2182 
2183  // Copy persistent_dst_buffer → new_frame buffer
2184  memcpy(
2185  new_frame->data[0],
2186  persistent_dst_buffer,
2187  static_cast<size_t>(bytes_final)
2188  );
2189 
2190  // Queue the deep‐copied frame for encoding
2191  add_avframe(frame, new_frame);
2192 }
2193 
2194 // write video frame
2195 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *frame_final) {
2196 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
2197  // FFmpeg 4.0+
2198  ZmqLogger::Instance()->AppendDebugMethod(
2199  "FFmpegWriter::write_video_packet",
2200  "frame->number", frame->number,
2201  "oc->oformat->flags", oc->oformat->flags);
2202 
2203  if (AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO) {
2204 #else
2205  // TODO: Should we have moved away from oc->oformat->flags / AVFMT_RAWPICTURE
2206  // on ffmpeg < 4.0 as well?
2207  // Does AV_CODEC_ID_RAWVIDEO not work in ffmpeg 3.x?
2208  ZmqLogger::Instance()->AppendDebugMethod(
2209  "FFmpegWriter::write_video_packet",
2210  "frame->number", frame->number,
2211  "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE);
2212 
2213  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
2214 #endif
2215  // Raw video case.
2216 #if IS_FFMPEG_3_2
2217  AVPacket* pkt = av_packet_alloc();
2218 #else
2219  AVPacket* pkt;
2220  av_init_packet(pkt);
2221 #endif
2222 
2223  av_packet_from_data(
2224  pkt, frame_final->data[0],
2225  frame_final->linesize[0] * frame_final->height);
2226 
2227  pkt->flags |= AV_PKT_FLAG_KEY;
2228  pkt->stream_index = video_st->index;
2229 
2230  // Set PTS (in frames and scaled to the codec's timebase)
2231  pkt->pts = video_timestamp;
2232 
2233  /* write the compressed frame in the media file */
2234  int error_code = av_interleaved_write_frame(oc, pkt);
2235  if (error_code < 0) {
2236  ZmqLogger::Instance()->AppendDebugMethod(
2237  "FFmpegWriter::write_video_packet ERROR ["
2238  + av_err2string(error_code) + "]",
2239  "error_code", error_code);
2240  return false;
2241  }
2242 
2243  // Deallocate packet
2244  AV_FREE_PACKET(pkt);
2245 
2246  } else
2247  {
2248 
2249 #if IS_FFMPEG_3_2
2250  AVPacket* pkt = av_packet_alloc();
2251 #else
2252  AVPacket* pkt;
2253  av_init_packet(pkt);
2254 #endif
2255  pkt->data = NULL;
2256  pkt->size = 0;
2257  pkt->pts = pkt->dts = AV_NOPTS_VALUE;
2258 
2259  // Assign the initial AVFrame PTS from the frame counter
2260  frame_final->pts = video_timestamp;
2261 #if USE_HW_ACCEL
2262  if (hw_en_on && hw_en_supported) {
2263  if (!(hw_frame = av_frame_alloc())) {
2264  std::clog << "Error code: av_hwframe_alloc\n";
2265  }
2266  if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) {
2267  std::clog << "Error code: av_hwframe_get_buffer\n";
2268  }
2269  if (!hw_frame->hw_frames_ctx) {
2270  std::clog << "Error hw_frames_ctx.\n";
2271  }
2272  hw_frame->format = AV_PIX_FMT_NV12;
2273  if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) {
2274  std::clog << "Error while transferring frame data to surface.\n";
2275  }
2276  av_frame_copy_props(hw_frame, frame_final);
2277  }
2278 #endif // USE_HW_ACCEL
2279  /* encode the image */
2280  int got_packet_ptr = 0;
2281  int error_code = 0;
2282 #if IS_FFMPEG_3_2
2283  // Write video packet
2284  int ret;
2285 
2286  #if USE_HW_ACCEL
2287  if (hw_en_on && hw_en_supported) {
2288  ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!!
2289  } else
2290  #endif // USE_HW_ACCEL
2291  {
2292  ret = avcodec_send_frame(video_codec_ctx, frame_final);
2293  }
2294  error_code = ret;
2295  if (ret < 0 ) {
2296  ZmqLogger::Instance()->AppendDebugMethod(
2297  "FFmpegWriter::write_video_packet (Frame not sent)");
2298  if (ret == AVERROR(EAGAIN) ) {
2299  std::clog << "Frame EAGAIN\n";
2300  }
2301  if (ret == AVERROR_EOF ) {
2302  std::clog << "Frame AVERROR_EOF\n";
2303  }
2304  avcodec_send_frame(video_codec_ctx, NULL);
2305  }
2306  else {
2307  while (ret >= 0) {
2308  ret = avcodec_receive_packet(video_codec_ctx, pkt);
2309 
2310  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
2311  got_packet_ptr = 0;
2312  break;
2313  }
2314  if (ret == 0) {
2315  got_packet_ptr = 1;
2316  break;
2317  }
2318  }
2319  }
2320 #else
2321  // Write video packet (older than FFmpeg 3.2)
2322  error_code = avcodec_encode_video2(video_codec_ctx, pkt, frame_final, &got_packet_ptr);
2323  if (error_code != 0) {
2324  ZmqLogger::Instance()->AppendDebugMethod(
2325  "FFmpegWriter::write_video_packet ERROR ["
2326  + av_err2string(error_code) + "]",
2327  "error_code", error_code);
2328  }
2329  if (got_packet_ptr == 0) {
2330  ZmqLogger::Instance()->AppendDebugMethod(
2331  "FFmpegWriter::write_video_packet (Frame gotpacket error)");
2332  }
2333 #endif // IS_FFMPEG_3_2
2334 
2335  /* if zero size, it means the image was buffered */
2336  if (error_code == 0 && got_packet_ptr) {
2337  // set the timestamp
2338  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
2339  pkt->stream_index = video_st->index;
2340 
2341  /* write the compressed frame in the media file */
2342  int result = av_interleaved_write_frame(oc, pkt);
2343  if (result < 0) {
2344  ZmqLogger::Instance()->AppendDebugMethod(
2345  "FFmpegWriter::write_video_packet ERROR ["
2346  + av_err2string(result) + "]",
2347  "result", result);
2348  return false;
2349  }
2350  }
2351 
2352  // Deallocate packet
2353  AV_FREE_PACKET(pkt);
2354 #if USE_HW_ACCEL
2355  if (hw_en_on && hw_en_supported) {
2356  if (hw_frame) {
2357  av_frame_free(&hw_frame);
2358  hw_frame = NULL;
2359  }
2360  }
2361 #endif // USE_HW_ACCEL
2362  }
2363 
2364  // Increment PTS (in frames and scaled to the codec's timebase)
2365  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
2366 
2367  // Success
2368  return true;
2369 }
2370 
2371 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
2372 void FFmpegWriter::OutputStreamInfo() {
2373  // output debug info
2374  av_dump_format(oc, 0, path.c_str(), 1);
2375 }
2376 
2377 // Set audio resample options
2378 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
2379  original_sample_rate = sample_rate;
2380  original_channels = channels;
2381 }
2382 
2383 // In FFmpegWriter.cpp
2384 void FFmpegWriter::AddSphericalMetadata(const std::string& projection, float yaw_deg, float pitch_deg, float roll_deg) {
2385  if (!oc) return;
2386  if (!info.has_video || !video_st) return;
2387 
2388  // Allow movenc.c to write out the sv3d atom
2389  oc->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
2390 
2391 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 0, 0)
2392  // Map the projection name to the enum (defaults to equirectangular)
2393  int proj = av_spherical_from_name(projection.c_str());
2394  if (proj < 0)
2395  proj = AV_SPHERICAL_EQUIRECTANGULAR;
2396 
2397  // Allocate the side‐data structure
2398  size_t sd_size = 0;
2399  AVSphericalMapping* map = av_spherical_alloc(&sd_size);
2400  if (!map) return;
2401 
2402  // Populate it
2403  map->projection = static_cast<AVSphericalProjection>(proj);
2404  // yaw/pitch/roll are 16.16 fixed point
2405  map->yaw = static_cast<int32_t>(yaw_deg * (1 << 16));
2406  map->pitch = static_cast<int32_t>(pitch_deg * (1 << 16));
2407  map->roll = static_cast<int32_t>(roll_deg * (1 << 16));
2408 
2409  ffmpeg_stream_add_side_data(video_st, AV_PKT_DATA_SPHERICAL,
2410  reinterpret_cast<uint8_t*>(map), sd_size);
2411 #endif
2412 }
AUDIO_PACKET_ENCODING_SIZE
#define AUDIO_PACKET_ENCODING_SIZE
Definition: FFmpegUtilities.h:90
Settings.h
Header file for global Settings class.
openshot::AUDIO_STREAM
@ AUDIO_STREAM
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:30
hw_frame
AVFrame * hw_frame
Definition: FFmpegWriter.cpp:43
openshot::InvalidFormat
Exception when no valid format is found for a file.
Definition: Exceptions.h:208
PIX_FMT_RGB24
#define PIX_FMT_RGB24
Definition: FFmpegUtilities.h:116
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:317
openshot::InvalidSampleRate
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:253
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::WriterInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:43
FFmpegWriter.h
Header file for FFmpegWriter class.
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:178
openshot::FFmpegWriter::ResampleAudio
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
Definition: FFmpegWriter.cpp:2378
openshot::WriterInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: WriterBase.h:45
openshot::WriterClosed
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:421
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:326
openshot::FFmpegWriter::OutputStreamInfo
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
Definition: FFmpegWriter.cpp:2372
PixelFormat
#define PixelFormat
Definition: FFmpegUtilities.h:107
openshot::ReaderBase::GetFrame
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:309
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:258
openshot::WriterInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:42
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
AV_OPTION_SET
#define AV_OPTION_SET(av_stream, priv_data, name, value, avcodec)
Definition: FFmpegUtilities.h:331
openshot::WriterInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:53
openshot::WriterInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:55
AV_COPY_PARAMS_FROM_CONTEXT
AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx)
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
AV_GET_CODEC_FROM_STREAM
#define AV_GET_CODEC_FROM_STREAM(av_stream, codec_in)
Definition: FFmpegUtilities.h:320
openshot::FFmpegWriter::FFmpegWriter
FFmpegWriter(const std::string &path)
Constructor for FFmpegWriter. Throws an exception on failure to open path.
Definition: FFmpegWriter.cpp:76
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:313
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:314
openshot::FFmpegWriter::Open
void Open()
Open writer.
Definition: FFmpegWriter.cpp:96
openshot::FFmpegWriter::SetVideoOptions
void SetVideoOptions(bool has_video, std::string codec, openshot::Fraction fps, int width, int height, openshot::Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
Definition: FFmpegWriter.cpp:165
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
AV_GET_CODEC_PAR_CONTEXT
#define AV_GET_CODEC_PAR_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:319
openshot::WriterInfo::width
int width
The width of the video (in pixels)
Definition: WriterBase.h:40
hw_en_on
int hw_en_on
Definition: FFmpegWriter.cpp:38
openshot::WriterInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:52
openshot::WriterInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:49
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:321
openshot::Settings::HW_EN_DEVICE_SET
int HW_EN_DEVICE_SET
Which GPU to use to encode (0 is the first)
Definition: Settings.h:95
openshot::WriterInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: WriterBase.h:44
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::WriterInfo::top_field_first
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:51
if
if(!codec) codec
AV_SET_FILENAME
#define AV_SET_FILENAME(oc, f)
Definition: FFmpegUtilities.h:307
AV_GET_IMAGE_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Definition: FFmpegUtilities.h:325
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
mux_dict
AVDictionary * mux_dict
Definition: FFmpegWriter.cpp:35
openshot::ErrorEncodingVideo
Exception when encoding audio packet.
Definition: Exceptions.h:148
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::FFmpegWriter::AddSphericalMetadata
void AddSphericalMetadata(const std::string &projection="equirectangular", float yaw_deg=0.0f, float pitch_deg=0.0f, float roll_deg=0.0f)
Add spherical (360°) video metadata to the video stream.
Definition: FFmpegWriter.cpp:2384
openshot::FFmpegWriter::SetOption
void SetOption(openshot::StreamType stream, std::string name, std::string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
Definition: FFmpegWriter.cpp:335
openshot::Fraction::Reduce
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:65
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:312
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:261
openshot::WriterInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:56
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:354
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:263
hw_en_av_pix_fmt
AVPixelFormat hw_en_av_pix_fmt
Definition: FFmpegWriter.cpp:40
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:43
openshot::VIDEO_STREAM
@ VIDEO_STREAM
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:29
openshot::WriterInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of video & audio metadata.
Definition: WriterBase.h:59
path
path
Definition: FFmpegWriter.cpp:1474
Frame.h
Header file for Frame class.
ALLOC_CODEC_CTX
#define ALLOC_CODEC_CTX(ctx, codec, stream)
Definition: FFmpegUtilities.h:334
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:193
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::FFmpegWriter::WriteFrame
void WriteFrame(std::shared_ptr< openshot::Frame > frame)
Add a frame to the stack waiting to be encoded.
Definition: FFmpegWriter.cpp:672
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::WriterInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:34
openshot::WriterInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:35
PIX_FMT_YUV420P
#define PIX_FMT_YUV420P
Definition: FFmpegUtilities.h:119
PIX_FMT_YUV444P
#define PIX_FMT_YUV444P
Definition: FFmpegUtilities.h:122
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:316
openshot::FFmpegWriter::Close
void Close()
Close the writer.
Definition: FFmpegWriter.cpp:962
openshot::FFmpegWriter::IsValidCodec
static bool IsValidCodec(std::string codec_name)
Determine if codec name is valid.
Definition: FFmpegWriter.cpp:604
openshot::WriterInfo::height
int height
The height of the video (in pixels)
Definition: WriterBase.h:39
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:315
OpenMPUtilities.h
Header file for OpenMPUtilities (set some common macros)
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:262
openshot::FFmpegWriter::WriteTrailer
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
Definition: FFmpegWriter.cpp:749
PIX_FMT_NONE
#define PIX_FMT_NONE
Definition: FFmpegUtilities.h:113
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
openshot::ReaderBase
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:75
openshot::WriterInfo::interlaced_frame
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:50
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::WriterInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:46
openshot::WriterInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:54
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:163
AV_OPTION_FIND
#define AV_OPTION_FIND(priv_data, name)
Definition: FFmpegUtilities.h:330
codec
codec
Definition: FFmpegWriter.cpp:1470
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
hw_en_av_device_type
AVHWDeviceType hw_en_av_device_type
Definition: FFmpegWriter.cpp:41
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:260
openshot::FFmpegWriter::SetAudioOptions
void SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, openshot::ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Definition: FFmpegWriter.cpp:288
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:304
AV_OUTPUT_CONTEXT
#define AV_OUTPUT_CONTEXT(output_context, path)
Definition: FFmpegUtilities.h:328
hw_en_supported
int hw_en_supported
Definition: FFmpegWriter.cpp:39
openshot::StreamType
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:28
openshot::FFmpegWriter::PrepareStreams
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
Definition: FFmpegWriter.cpp:613
openshot::InvalidOptions
Exception when invalid encoding options are used.
Definition: Exceptions.h:238
openshot::WriterBase::info
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:76
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:291
MY_INPUT_BUFFER_PADDING_SIZE
#define MY_INPUT_BUFFER_PADDING_SIZE
Definition: FFmpegUtilities.h:308
AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
Definition: FFmpegUtilities.h:83
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1485
Exceptions.h
Header file for all Exception classes.
openshot::FFmpegWriter::WriteHeader
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
Definition: FFmpegWriter.cpp:630
Generated by   doxygen 1.8.17