video_frame_extractor.cc 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/android/video_frame_extractor.h"
  5. #include "base/android/build_info.h"
  6. #include "base/bind.h"
  7. #include "base/threading/thread.h"
  8. #include "base/threading/thread_task_runner_handle.h"
  9. #include "media/base/android/media_codec_bridge_impl.h"
  10. #include "media/base/data_source.h"
  11. #include "media/ffmpeg/ffmpeg_common.h"
  12. #include "media/filters/blocking_url_protocol.h"
  13. #include "media/filters/ffmpeg_bitstream_converter.h"
  14. #include "media/filters/ffmpeg_glue.h"
  15. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  16. #include "media/filters/ffmpeg_h265_to_annex_b_bitstream_converter.h"
  17. #endif
  18. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  19. #include "media/base/android/extract_sps_and_pps.h"
  20. #include "media/filters/ffmpeg_aac_bitstream_converter.h"
  21. #include "media/filters/ffmpeg_h264_to_annex_b_bitstream_converter.h"
  22. #endif
  23. namespace media {
  24. VideoFrameExtractor::VideoFrameExtractor(DataSource* data_source)
  25. : data_source_(data_source), video_stream_index_(-1) {}
  26. VideoFrameExtractor::~VideoFrameExtractor() = default;
  27. void VideoFrameExtractor::Start(VideoFrameCallback video_frame_callback) {
  28. video_frame_callback_ = std::move(video_frame_callback);
  29. protocol_ = std::make_unique<BlockingUrlProtocol>(
  30. data_source_, base::BindRepeating(&VideoFrameExtractor::OnError,
  31. weak_factory_.GetWeakPtr()));
  32. glue_ = std::make_unique<FFmpegGlue>(protocol_.get());
  33. // This will gradually read media data through |data_source_|.
  34. if (!glue_->OpenContext()) {
  35. OnError();
  36. return;
  37. }
  38. // Extract the video stream.
  39. AVFormatContext* format_context = glue_->format_context();
  40. for (unsigned int i = 0; i < format_context->nb_streams; ++i) {
  41. AVStream* stream = format_context->streams[i];
  42. if (!stream)
  43. continue;
  44. const AVCodecParameters* codec_parameters = stream->codecpar;
  45. const AVMediaType codec_type = codec_parameters->codec_type;
  46. // Pick the first video stream.
  47. if (codec_type == AVMEDIA_TYPE_VIDEO) {
  48. video_stream_ = stream;
  49. video_stream_index_ = i;
  50. DCHECK_EQ(video_stream_index_, stream->index);
  51. break;
  52. }
  53. }
  54. if (!video_stream_) {
  55. OnError();
  56. return;
  57. }
  58. // Get the config for decoding the video frame later.
  59. if (!AVStreamToVideoDecoderConfig(video_stream_, &video_config_)) {
  60. OnError();
  61. return;
  62. }
  63. auto packet = ReadVideoFrame();
  64. if (!packet) {
  65. OnError();
  66. return;
  67. }
  68. ConvertPacket(packet.get());
  69. NotifyComplete(
  70. std::vector<uint8_t>(packet->data, packet->data + packet->size),
  71. video_config_);
  72. }
  73. void VideoFrameExtractor::ConvertPacket(AVPacket* packet) {
  74. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  75. DCHECK(video_stream_ && video_stream_->codecpar);
  76. // TODO(xingliu): Create the bitstream converter in an utility function. This
  77. // logic is shared with FFmpegDemuxer.
  78. switch (video_stream_->codecpar->codec_id) {
  79. case AV_CODEC_ID_H264:
  80. video_config_.SetExtraData(std::vector<uint8_t>());
  81. bitstream_converter_.reset(
  82. new FFmpegH264ToAnnexBBitstreamConverter(video_stream_->codecpar));
  83. break;
  84. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  85. case AV_CODEC_ID_HEVC:
  86. bitstream_converter_.reset(
  87. new FFmpegH265ToAnnexBBitstreamConverter(video_stream_->codecpar));
  88. break;
  89. #endif
  90. case AV_CODEC_ID_AAC:
  91. bitstream_converter_.reset(
  92. new FFmpegAACBitstreamConverter(video_stream_->codecpar));
  93. break;
  94. default:
  95. break;
  96. }
  97. if (bitstream_converter_)
  98. bitstream_converter_->ConvertPacket(packet);
  99. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  100. }
  101. ScopedAVPacket VideoFrameExtractor::ReadVideoFrame() {
  102. AVFormatContext* format_context = glue_->format_context();
  103. auto packet = ScopedAVPacket::Allocate();
  104. while (av_read_frame(format_context, packet.get()) >= 0) {
  105. // Skip frames from streams other than video.
  106. if (packet->stream_index != video_stream_index_)
  107. continue;
  108. DCHECK(packet->flags & AV_PKT_FLAG_KEY);
  109. return packet;
  110. }
  111. return {};
  112. }
  113. void VideoFrameExtractor::NotifyComplete(std::vector<uint8_t> encoded_frame,
  114. const VideoDecoderConfig& config) {
  115. // Return the encoded video key frame.
  116. DCHECK(video_frame_callback_);
  117. std::move(video_frame_callback_).Run(true, std::move(encoded_frame), config);
  118. }
  119. void VideoFrameExtractor::OnError() {
  120. DCHECK(video_frame_callback_);
  121. std::move(video_frame_callback_)
  122. .Run(false, std::vector<uint8_t>(), VideoDecoderConfig());
  123. }
  124. } // namespace media