avc.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/formats/mp4/avc.h"
  5. #include <algorithm>
  6. #include <memory>
  7. #include <utility>
  8. #include "base/logging.h"
  9. #include "media/base/decrypt_config.h"
  10. #include "media/formats/mp4/box_definitions.h"
  11. #include "media/formats/mp4/box_reader.h"
  12. #include "media/video/h264_parser.h"
  13. namespace media {
  14. namespace mp4 {
  15. static constexpr uint8_t kAnnexBStartCode[] = {0, 0, 0, 1};
  16. static constexpr int kAnnexBStartCodeSize = 4;
  17. static bool ConvertAVCToAnnexBInPlaceForLengthSize4(std::vector<uint8_t>* buf) {
  18. const size_t kLengthSize = 4;
  19. size_t pos = 0;
  20. while (buf->size() > kLengthSize && buf->size() - kLengthSize > pos) {
  21. uint32_t nal_length = (*buf)[pos];
  22. nal_length = (nal_length << 8) + (*buf)[pos+1];
  23. nal_length = (nal_length << 8) + (*buf)[pos+2];
  24. nal_length = (nal_length << 8) + (*buf)[pos+3];
  25. if (nal_length == 0) {
  26. DVLOG(3) << "nal_length is 0";
  27. return false;
  28. }
  29. std::copy(kAnnexBStartCode, kAnnexBStartCode + kAnnexBStartCodeSize,
  30. buf->begin() + pos);
  31. pos += kLengthSize + nal_length;
  32. }
  33. return pos == buf->size();
  34. }
  35. // static
  36. int AVC::FindSubsampleIndex(const std::vector<uint8_t>& buffer,
  37. const std::vector<SubsampleEntry>* subsamples,
  38. const uint8_t* ptr) {
  39. DCHECK(ptr >= &buffer[0]);
  40. DCHECK(ptr <= &buffer[buffer.size()-1]);
  41. if (!subsamples || subsamples->empty())
  42. return 0;
  43. const uint8_t* p = &buffer[0];
  44. for (size_t i = 0; i < subsamples->size(); ++i) {
  45. p += (*subsamples)[i].clear_bytes + (*subsamples)[i].cypher_bytes;
  46. if (p > ptr)
  47. return i;
  48. }
  49. NOTREACHED();
  50. return 0;
  51. }
  52. // static
  53. bool AVC::ConvertFrameToAnnexB(size_t length_size,
  54. std::vector<uint8_t>* buffer,
  55. std::vector<SubsampleEntry>* subsamples) {
  56. RCHECK(length_size == 1 || length_size == 2 || length_size == 4);
  57. DVLOG(5) << __func__ << " length_size=" << length_size
  58. << " buffer->size()=" << buffer->size()
  59. << " subsamples=" << (subsamples ? subsamples->size() : 0);
  60. if (length_size == 4)
  61. return ConvertAVCToAnnexBInPlaceForLengthSize4(buffer);
  62. std::vector<uint8_t> temp;
  63. temp.swap(*buffer);
  64. buffer->reserve(temp.size() + 32);
  65. size_t pos = 0;
  66. while (temp.size() > length_size && temp.size() - length_size > pos) {
  67. size_t nal_length = temp[pos];
  68. if (length_size == 2) nal_length = (nal_length << 8) + temp[pos+1];
  69. pos += length_size;
  70. if (nal_length == 0) {
  71. DVLOG(3) << "nal_length is 0";
  72. return false;
  73. }
  74. RCHECK(temp.size() >= nal_length && temp.size() - nal_length >= pos);
  75. buffer->insert(buffer->end(), kAnnexBStartCode,
  76. kAnnexBStartCode + kAnnexBStartCodeSize);
  77. if (subsamples && !subsamples->empty()) {
  78. uint8_t* buffer_pos = &(*(buffer->end() - kAnnexBStartCodeSize));
  79. int subsample_index = FindSubsampleIndex(*buffer, subsamples, buffer_pos);
  80. // We've replaced NALU size value with an AnnexB start code.
  81. int size_adjustment = kAnnexBStartCodeSize - length_size;
  82. (*subsamples)[subsample_index].clear_bytes += size_adjustment;
  83. }
  84. buffer->insert(buffer->end(), temp.begin() + pos,
  85. temp.begin() + pos + nal_length);
  86. pos += nal_length;
  87. }
  88. return pos == temp.size();
  89. }
  90. // static
  91. bool AVC::InsertParamSetsAnnexB(const AVCDecoderConfigurationRecord& avc_config,
  92. std::vector<uint8_t>* buffer,
  93. std::vector<SubsampleEntry>* subsamples) {
  94. std::unique_ptr<H264Parser> parser(new H264Parser());
  95. const uint8_t* start = &(*buffer)[0];
  96. parser->SetEncryptedStream(start, buffer->size(), *subsamples);
  97. H264NALU nalu;
  98. if (parser->AdvanceToNextNALU(&nalu) != H264Parser::kOk)
  99. return false;
  100. std::vector<uint8_t>::iterator config_insert_point = buffer->begin();
  101. if (nalu.nal_unit_type == H264NALU::kAUD) {
  102. // Move insert point to just after the AUD.
  103. config_insert_point += (nalu.data + nalu.size) - start;
  104. }
  105. // Clear |parser| and |start| since they aren't needed anymore and
  106. // will hold stale pointers once the insert happens.
  107. parser.reset();
  108. start = NULL;
  109. std::vector<uint8_t> param_sets;
  110. RCHECK(AVC::ConvertConfigToAnnexB(avc_config, &param_sets));
  111. if (subsamples && !subsamples->empty()) {
  112. if (config_insert_point != buffer->end()) {
  113. int subsample_index =
  114. FindSubsampleIndex(*buffer, subsamples, &(*config_insert_point));
  115. // Update the size of the subsample where SPS/PPS is to be inserted.
  116. (*subsamples)[subsample_index].clear_bytes += param_sets.size();
  117. } else {
  118. int subsample_index = (*subsamples).size() - 1;
  119. if ((*subsamples)[subsample_index].cypher_bytes == 0) {
  120. // Extend the last clear range to include the inserted data.
  121. (*subsamples)[subsample_index].clear_bytes += param_sets.size();
  122. } else {
  123. // Append a new subsample to cover the inserted data.
  124. (*subsamples).emplace_back(param_sets.size(), 0);
  125. }
  126. }
  127. }
  128. buffer->insert(config_insert_point,
  129. param_sets.begin(), param_sets.end());
  130. return true;
  131. }
  132. // static
  133. bool AVC::ConvertConfigToAnnexB(const AVCDecoderConfigurationRecord& avc_config,
  134. std::vector<uint8_t>* buffer) {
  135. DCHECK(buffer->empty());
  136. buffer->clear();
  137. int total_size = 0;
  138. for (size_t i = 0; i < avc_config.sps_list.size(); i++)
  139. total_size += avc_config.sps_list[i].size() + kAnnexBStartCodeSize;
  140. for (size_t i = 0; i < avc_config.pps_list.size(); i++)
  141. total_size += avc_config.pps_list[i].size() + kAnnexBStartCodeSize;
  142. buffer->reserve(total_size);
  143. for (size_t i = 0; i < avc_config.sps_list.size(); i++) {
  144. buffer->insert(buffer->end(), kAnnexBStartCode,
  145. kAnnexBStartCode + kAnnexBStartCodeSize);
  146. buffer->insert(buffer->end(), avc_config.sps_list[i].begin(),
  147. avc_config.sps_list[i].end());
  148. }
  149. for (size_t i = 0; i < avc_config.pps_list.size(); i++) {
  150. buffer->insert(buffer->end(), kAnnexBStartCode,
  151. kAnnexBStartCode + kAnnexBStartCodeSize);
  152. buffer->insert(buffer->end(), avc_config.pps_list[i].begin(),
  153. avc_config.pps_list[i].end());
  154. }
  155. return true;
  156. }
  157. // static
  158. BitstreamConverter::AnalysisResult AVC::AnalyzeAnnexB(
  159. const uint8_t* buffer,
  160. size_t size,
  161. const std::vector<SubsampleEntry>& subsamples) {
  162. DVLOG(3) << __func__;
  163. DCHECK(buffer);
  164. BitstreamConverter::AnalysisResult result;
  165. result.is_conformant = false; // Will change if needed before return.
  166. if (size == 0) {
  167. result.is_conformant = true;
  168. return result;
  169. }
  170. H264Parser parser;
  171. parser.SetEncryptedStream(buffer, size, subsamples);
  172. typedef enum {
  173. kAUDAllowed,
  174. kBeforeFirstVCL, // VCL == nal_unit_types 1-5
  175. kAfterFirstVCL,
  176. kEOStreamAllowed,
  177. kNoMoreDataAllowed,
  178. } NALUOrderState;
  179. H264NALU nalu;
  180. NALUOrderState order_state = kAUDAllowed;
  181. int last_nalu_type = H264NALU::kUnspecified;
  182. bool done = false;
  183. while (!done) {
  184. switch (parser.AdvanceToNextNALU(&nalu)) {
  185. case H264Parser::kOk:
  186. DVLOG(3) << "nal_unit_type " << nalu.nal_unit_type;
  187. switch (nalu.nal_unit_type) {
  188. case H264NALU::kAUD:
  189. if (order_state > kAUDAllowed) {
  190. DVLOG(1) << "Unexpected AUD in order_state " << order_state;
  191. return result;
  192. }
  193. order_state = kBeforeFirstVCL;
  194. break;
  195. case H264NALU::kSEIMessage:
  196. case H264NALU::kPrefix:
  197. case H264NALU::kSubsetSPS:
  198. case H264NALU::kDPS:
  199. case H264NALU::kReserved17:
  200. case H264NALU::kReserved18:
  201. case H264NALU::kPPS:
  202. case H264NALU::kSPS:
  203. if (order_state > kBeforeFirstVCL) {
  204. DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
  205. << " in order_state " << order_state;
  206. return result;
  207. }
  208. order_state = kBeforeFirstVCL;
  209. break;
  210. case H264NALU::kSPSExt:
  211. if (last_nalu_type != H264NALU::kSPS) {
  212. DVLOG(1) << "SPS extension does not follow an SPS.";
  213. return result;
  214. }
  215. break;
  216. case H264NALU::kNonIDRSlice:
  217. case H264NALU::kSliceDataA:
  218. case H264NALU::kSliceDataB:
  219. case H264NALU::kSliceDataC:
  220. case H264NALU::kIDRSlice:
  221. if (order_state > kAfterFirstVCL) {
  222. DVLOG(1) << "Unexpected VCL in order_state " << order_state;
  223. return result;
  224. }
  225. if (!result.is_keyframe.has_value())
  226. result.is_keyframe = nalu.nal_unit_type == H264NALU::kIDRSlice;
  227. order_state = kAfterFirstVCL;
  228. break;
  229. case H264NALU::kCodedSliceAux:
  230. if (order_state != kAfterFirstVCL) {
  231. DVLOG(1) << "Unexpected extension in order_state " << order_state;
  232. return result;
  233. }
  234. break;
  235. case H264NALU::kEOSeq:
  236. if (order_state != kAfterFirstVCL) {
  237. DVLOG(1) << "Unexpected EOSeq in order_state " << order_state;
  238. return result;
  239. }
  240. order_state = kEOStreamAllowed;
  241. break;
  242. case H264NALU::kEOStream:
  243. if (order_state < kAfterFirstVCL) {
  244. DVLOG(1) << "Unexpected EOStream in order_state " << order_state;
  245. return result;
  246. }
  247. order_state = kNoMoreDataAllowed;
  248. break;
  249. case H264NALU::kFiller:
  250. case H264NALU::kUnspecified:
  251. if (!(order_state >= kAfterFirstVCL &&
  252. order_state < kEOStreamAllowed)) {
  253. DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
  254. << " in order_state " << order_state;
  255. return result;
  256. }
  257. break;
  258. default:
  259. DCHECK_GE(nalu.nal_unit_type, 20);
  260. if (nalu.nal_unit_type >= 20 && nalu.nal_unit_type <= 31 &&
  261. order_state != kAfterFirstVCL) {
  262. DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
  263. << " in order_state " << order_state;
  264. return result;
  265. }
  266. }
  267. last_nalu_type = nalu.nal_unit_type;
  268. break;
  269. case H264Parser::kInvalidStream:
  270. return result;
  271. case H264Parser::kUnsupportedStream:
  272. NOTREACHED() << "AdvanceToNextNALU() returned kUnsupportedStream!";
  273. return result;
  274. case H264Parser::kEOStream:
  275. done = true;
  276. }
  277. }
  278. if (order_state < kAfterFirstVCL)
  279. return result;
  280. result.is_conformant = true;
  281. DCHECK(result.is_keyframe.has_value());
  282. return result;
  283. }
  284. AVCBitstreamConverter::AVCBitstreamConverter(
  285. std::unique_ptr<AVCDecoderConfigurationRecord> avc_config)
  286. : avc_config_(std::move(avc_config)) {
  287. DCHECK(avc_config_);
  288. }
  289. AVCBitstreamConverter::~AVCBitstreamConverter() = default;
  290. bool AVCBitstreamConverter::ConvertAndAnalyzeFrame(
  291. std::vector<uint8_t>* frame_buf,
  292. bool is_keyframe,
  293. std::vector<SubsampleEntry>* subsamples,
  294. AnalysisResult* analysis_result) const {
  295. // Convert the AVC NALU length fields to Annex B headers, as expected by
  296. // decoding libraries. Since this may enlarge the size of the buffer, we also
  297. // update the clear byte count for each subsample if encryption is used to
  298. // account for the difference in size between the length prefix and Annex B
  299. // start code.
  300. RCHECK(AVC::ConvertFrameToAnnexB(avc_config_->length_size, frame_buf,
  301. subsamples));
  302. // |is_keyframe| may be incorrect. Analyze the frame to see if it is a
  303. // keyframe. |is_keyframe| will be used if the analysis is inconclusive.
  304. // Also, provide the analysis result to the caller via out parameter
  305. // |analysis_result|.
  306. *analysis_result = Analyze(frame_buf, subsamples);
  307. if (analysis_result->is_keyframe.value_or(is_keyframe)) {
  308. // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
  309. // a frame. If subsample info is present, we also update the clear byte
  310. // count for that first subsample.
  311. RCHECK(AVC::InsertParamSetsAnnexB(*avc_config_, frame_buf, subsamples));
  312. }
  313. return true;
  314. }
  315. BitstreamConverter::AnalysisResult AVCBitstreamConverter::Analyze(
  316. std::vector<uint8_t>* frame_buf,
  317. std::vector<SubsampleEntry>* subsamples) const {
  318. return AVC::AnalyzeAnnexB(frame_buf->data(), frame_buf->size(), *subsamples);
  319. }
  320. } // namespace mp4
  321. } // namespace media