avc_unittest.cc 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/formats/mp4/avc.h"
  5. #include <stddef.h>
  6. #include <stdint.h>
  7. #include <string.h>
  8. #include <ostream>
  9. #include "base/strings/string_split.h"
  10. #include "base/strings/string_util.h"
  11. #include "media/base/decrypt_config.h"
  12. #include "media/base/stream_parser_buffer.h"
  13. #include "media/formats/mp4/bitstream_converter.h"
  14. #include "media/formats/mp4/box_definitions.h"
  15. #include "media/formats/mp4/nalu_test_helper.h"
  16. #include "media/video/h264_parser.h"
  17. #include "testing/gtest/include/gtest/gtest.h"
  18. #include "third_party/abseil-cpp/absl/types/optional.h"
  19. namespace media {
  20. namespace mp4 {
  21. static const uint8_t kNALU1[] = {0x01, 0x02, 0x03};
  22. static const uint8_t kNALU2[] = {0x04, 0x05, 0x06, 0x07};
  23. static const uint8_t kExpected[] = {0x00, 0x00, 0x00, 0x01, 0x01,
  24. 0x02, 0x03, 0x00, 0x00, 0x00,
  25. 0x01, 0x04, 0x05, 0x06, 0x07};
  26. static const uint8_t kExpectedParamSets[] = {
  27. 0x00, 0x00, 0x00, 0x01, 0x67, 0x12, 0x00, 0x00, 0x00, 0x01,
  28. 0x67, 0x34, 0x00, 0x00, 0x00, 0x01, 0x68, 0x56, 0x78};
  29. static std::string NALUTypeToString(int type) {
  30. switch (type) {
  31. case H264NALU::kNonIDRSlice:
  32. return "P";
  33. case H264NALU::kSliceDataA:
  34. return "SDA";
  35. case H264NALU::kSliceDataB:
  36. return "SDB";
  37. case H264NALU::kSliceDataC:
  38. return "SDC";
  39. case H264NALU::kIDRSlice:
  40. return "I";
  41. case H264NALU::kSEIMessage:
  42. return "SEI";
  43. case H264NALU::kSPS:
  44. return "SPS";
  45. case H264NALU::kSPSExt:
  46. return "SPSExt";
  47. case H264NALU::kPPS:
  48. return "PPS";
  49. case H264NALU::kAUD:
  50. return "AUD";
  51. case H264NALU::kEOSeq:
  52. return "EOSeq";
  53. case H264NALU::kEOStream:
  54. return "EOStr";
  55. case H264NALU::kFiller:
  56. return "FILL";
  57. case H264NALU::kPrefix:
  58. return "Prefix";
  59. case H264NALU::kSubsetSPS:
  60. return "SubsetSPS";
  61. case H264NALU::kDPS:
  62. return "DPS";
  63. case H264NALU::kUnspecified:
  64. case H264NALU::kReserved17:
  65. case H264NALU::kReserved18:
  66. case H264NALU::kCodedSliceAux:
  67. case H264NALU::kCodedSliceExtension:
  68. CHECK(false) << "Unexpected type: " << type;
  69. break;
  70. };
  71. return "UnsupportedType";
  72. }
  73. // Helper output operator, for debugging/testability.
  74. std::ostream& operator<<(std::ostream& os,
  75. const BitstreamConverter::AnalysisResult& r) {
  76. os << "{ is_conformant: "
  77. << (r.is_conformant.has_value()
  78. ? (r.is_conformant.value() ? "true" : "false")
  79. : "nullopt/unknown")
  80. << ", is_keyframe: "
  81. << (r.is_keyframe.has_value() ? (r.is_keyframe.value() ? "true" : "false")
  82. : "nullopt/unknown")
  83. << " }";
  84. return os;
  85. }
  86. static std::string AnnexBToString(
  87. const std::vector<uint8_t>& buffer,
  88. const std::vector<SubsampleEntry>& subsamples) {
  89. std::stringstream ss;
  90. H264Parser parser;
  91. parser.SetEncryptedStream(&buffer[0], buffer.size(), subsamples);
  92. H264NALU nalu;
  93. bool first = true;
  94. size_t current_subsample_index = 0;
  95. while (parser.AdvanceToNextNALU(&nalu) == H264Parser::kOk) {
  96. size_t subsample_index = AVC::FindSubsampleIndex(buffer, &subsamples,
  97. nalu.data);
  98. if (!first) {
  99. ss << (subsample_index == current_subsample_index ? "," : " ");
  100. } else {
  101. DCHECK_EQ(subsample_index, current_subsample_index);
  102. first = false;
  103. }
  104. ss << NALUTypeToString(nalu.nal_unit_type);
  105. current_subsample_index = subsample_index;
  106. }
  107. return ss.str();
  108. }
  109. class AVCConversionTest : public testing::TestWithParam<int> {
  110. protected:
  111. void WriteLength(int length_size, int length, std::vector<uint8_t>* buf) {
  112. DCHECK_GE(length, 0);
  113. DCHECK_LE(length, 255);
  114. for (int i = 1; i < length_size; i++)
  115. buf->push_back(0);
  116. buf->push_back(length);
  117. }
  118. void MakeInputForLength(int length_size, std::vector<uint8_t>* buf) {
  119. buf->clear();
  120. WriteLength(length_size, sizeof(kNALU1), buf);
  121. buf->insert(buf->end(), kNALU1, kNALU1 + sizeof(kNALU1));
  122. WriteLength(length_size, sizeof(kNALU2), buf);
  123. buf->insert(buf->end(), kNALU2, kNALU2 + sizeof(kNALU2));
  124. }
  125. };
  126. TEST_P(AVCConversionTest, ParseCorrectly) {
  127. std::vector<uint8_t> buf;
  128. std::vector<SubsampleEntry> subsamples;
  129. MakeInputForLength(GetParam(), &buf);
  130. EXPECT_TRUE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, &subsamples));
  131. BitstreamConverter::AnalysisResult expected;
  132. expected.is_conformant = true;
  133. expected.is_keyframe = false;
  134. EXPECT_PRED2(AnalysesMatch,
  135. AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
  136. expected);
  137. EXPECT_EQ(buf.size(), sizeof(kExpected));
  138. EXPECT_EQ(0, memcmp(kExpected, &buf[0], sizeof(kExpected)));
  139. EXPECT_EQ("P,SDC", AnnexBToString(buf, subsamples));
  140. }
  141. // Intentionally write NALU sizes that are larger than the buffer.
  142. TEST_P(AVCConversionTest, NALUSizeTooLarge) {
  143. std::vector<uint8_t> buf;
  144. WriteLength(GetParam(), 10 * sizeof(kNALU1), &buf);
  145. buf.insert(buf.end(), kNALU1, kNALU1 + sizeof(kNALU1));
  146. EXPECT_FALSE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, nullptr));
  147. }
  148. TEST_P(AVCConversionTest, NALUSizeIsZero) {
  149. std::vector<uint8_t> buf;
  150. WriteLength(GetParam(), 0, &buf);
  151. WriteLength(GetParam(), sizeof(kNALU1), &buf);
  152. buf.insert(buf.end(), kNALU1, kNALU1 + sizeof(kNALU1));
  153. WriteLength(GetParam(), 0, &buf);
  154. WriteLength(GetParam(), sizeof(kNALU2), &buf);
  155. buf.insert(buf.end(), kNALU2, kNALU2 + sizeof(kNALU2));
  156. EXPECT_FALSE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, nullptr));
  157. }
  158. TEST_P(AVCConversionTest, SubsampleSizesUpdatedAfterAnnexBConversion) {
  159. std::vector<uint8_t> buf;
  160. std::vector<SubsampleEntry> subsamples;
  161. SubsampleEntry subsample;
  162. // Write the first subsample, consisting of only one NALU
  163. WriteLength(GetParam(), sizeof(kNALU1), &buf);
  164. buf.insert(buf.end(), kNALU1, kNALU1 + sizeof(kNALU1));
  165. subsample.clear_bytes = GetParam() + sizeof(kNALU1);
  166. subsample.cypher_bytes = 0;
  167. subsamples.push_back(subsample);
  168. // Write the second subsample, containing two NALUs
  169. WriteLength(GetParam(), sizeof(kNALU1), &buf);
  170. buf.insert(buf.end(), kNALU1, kNALU1 + sizeof(kNALU1));
  171. WriteLength(GetParam(), sizeof(kNALU2), &buf);
  172. buf.insert(buf.end(), kNALU2, kNALU2 + sizeof(kNALU2));
  173. subsample.clear_bytes = 2*GetParam() + sizeof(kNALU1) + sizeof(kNALU2);
  174. subsample.cypher_bytes = 0;
  175. subsamples.push_back(subsample);
  176. // Write the third subsample, containing a single one-byte NALU
  177. WriteLength(GetParam(), 1, &buf);
  178. buf.push_back(0);
  179. subsample.clear_bytes = GetParam() + 1;
  180. subsample.cypher_bytes = 0;
  181. subsamples.push_back(subsample);
  182. EXPECT_TRUE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, &subsamples));
  183. EXPECT_EQ(subsamples.size(), 3u);
  184. EXPECT_EQ(subsamples[0].clear_bytes, 4 + sizeof(kNALU1));
  185. EXPECT_EQ(subsamples[0].cypher_bytes, 0u);
  186. EXPECT_EQ(subsamples[1].clear_bytes, 8 + sizeof(kNALU1) + sizeof(kNALU2));
  187. EXPECT_EQ(subsamples[1].cypher_bytes, 0u);
  188. EXPECT_EQ(subsamples[2].clear_bytes, 4 + 1u);
  189. EXPECT_EQ(subsamples[2].cypher_bytes, 0u);
  190. }
  191. TEST_P(AVCConversionTest, ParsePartial) {
  192. std::vector<uint8_t> buf;
  193. MakeInputForLength(GetParam(), &buf);
  194. buf.pop_back();
  195. EXPECT_FALSE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, nullptr));
  196. // This tests a buffer ending in the middle of a NAL length. For length size
  197. // of one, this can't happen, so we skip that case.
  198. if (GetParam() != 1) {
  199. MakeInputForLength(GetParam(), &buf);
  200. buf.erase(buf.end() - (sizeof(kNALU2) + 1), buf.end());
  201. EXPECT_FALSE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, nullptr));
  202. }
  203. }
  204. TEST_P(AVCConversionTest, ParseEmpty) {
  205. std::vector<uint8_t> buf;
  206. EXPECT_TRUE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, nullptr));
  207. EXPECT_EQ(0u, buf.size());
  208. }
  209. INSTANTIATE_TEST_SUITE_P(AVCConversionTestValues,
  210. AVCConversionTest,
  211. ::testing::Values(1, 2, 4));
  212. TEST_F(AVCConversionTest, ConvertConfigToAnnexB) {
  213. AVCDecoderConfigurationRecord avc_config;
  214. avc_config.sps_list.resize(2);
  215. avc_config.sps_list[0].push_back(0x67);
  216. avc_config.sps_list[0].push_back(0x12);
  217. avc_config.sps_list[1].push_back(0x67);
  218. avc_config.sps_list[1].push_back(0x34);
  219. avc_config.pps_list.resize(1);
  220. avc_config.pps_list[0].push_back(0x68);
  221. avc_config.pps_list[0].push_back(0x56);
  222. avc_config.pps_list[0].push_back(0x78);
  223. std::vector<uint8_t> buf;
  224. std::vector<SubsampleEntry> subsamples;
  225. EXPECT_TRUE(AVC::ConvertConfigToAnnexB(avc_config, &buf));
  226. EXPECT_EQ(0, memcmp(kExpectedParamSets, &buf[0],
  227. sizeof(kExpectedParamSets)));
  228. EXPECT_EQ("SPS,SPS,PPS", AnnexBToString(buf, subsamples));
  229. }
  230. // Verify that we can round trip string -> Annex B -> string.
  231. TEST_F(AVCConversionTest, StringConversionFunctions) {
  232. std::string str =
  233. "AUD SPS SPSExt SPS PPS SEI SEI Prefix I P FILL EOSeq EOStr";
  234. std::vector<uint8_t> buf;
  235. std::vector<SubsampleEntry> subsamples;
  236. AvcStringToAnnexB(str, &buf, &subsamples);
  237. BitstreamConverter::AnalysisResult expected;
  238. expected.is_conformant = true;
  239. expected.is_keyframe = true;
  240. EXPECT_PRED2(AnalysesMatch,
  241. AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
  242. expected);
  243. EXPECT_EQ(str, AnnexBToString(buf, subsamples));
  244. }
  245. TEST_F(AVCConversionTest, ValidAnnexBConstructs) {
  246. struct {
  247. const char* case_string;
  248. const bool is_keyframe;
  249. } test_cases[] = {
  250. {"I", true},
  251. {"I I I I", true},
  252. {"AUD I", true},
  253. {"AUD SPS PPS I", true},
  254. {"I EOSeq", true},
  255. {"I EOSeq EOStr", true},
  256. {"I EOStr", true},
  257. {"P", false},
  258. {"P P P P", false},
  259. {"AUD SPS PPS P", false},
  260. {"SEI SEI I", true},
  261. {"SEI SEI Prefix I", true},
  262. {"SPS SPSExt SPS PPS I P", true},
  263. {"Prefix SEI I", true},
  264. {"AUD,I", true},
  265. {"AUD,SEI I", true},
  266. {"AUD,SEI,SPS,PPS,I", true},
  267. // In reality, these might not always be conformant/valid, but assuming
  268. // they are, they're not keyframes because a non-IDR slice preceded the
  269. // IDR slice, if any.
  270. {"SDA SDB SDC", false},
  271. {"P I", false},
  272. {"SDA I", false},
  273. {"SDB I", false},
  274. {"SDC I", false},
  275. };
  276. for (size_t i = 0; i < std::size(test_cases); ++i) {
  277. std::vector<uint8_t> buf;
  278. std::vector<SubsampleEntry> subsamples;
  279. AvcStringToAnnexB(test_cases[i].case_string, &buf, NULL);
  280. BitstreamConverter::AnalysisResult expected;
  281. expected.is_conformant = true;
  282. expected.is_keyframe = test_cases[i].is_keyframe;
  283. EXPECT_PRED2(AnalysesMatch,
  284. AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
  285. expected)
  286. << "'" << test_cases[i].case_string << "' failed";
  287. }
  288. }
  289. TEST_F(AVCConversionTest, InvalidAnnexBConstructs) {
  290. struct {
  291. const char* case_string;
  292. const absl::optional<bool> is_keyframe;
  293. } test_cases[] = {
  294. // For these cases, lack of conformance is determined before detecting any
  295. // IDR or non-IDR slices, so the non-conformant frames' keyframe analysis
  296. // reports absl::nullopt (which means undetermined analysis result).
  297. {"AUD", absl::nullopt}, // No VCL present.
  298. {"AUD,SEI", absl::nullopt}, // No VCL present.
  299. {"SPS PPS", absl::nullopt}, // No VCL present.
  300. {"SPS PPS AUD I", absl::nullopt}, // Parameter sets must come after AUD.
  301. {"SPSExt SPS P", absl::nullopt}, // SPS must come before SPSExt.
  302. {"SPS PPS SPSExt P", absl::nullopt}, // SPSExt must follow an SPS.
  303. {"EOSeq", absl::nullopt}, // EOSeq must come after a VCL.
  304. {"EOStr", absl::nullopt}, // EOStr must come after a VCL.
  305. // For these cases, IDR slice is first VCL and is detected before
  306. // conformance failure, so the non-conformant frame is reported as a
  307. // keyframe.
  308. {"I EOStr EOSeq", true}, // EOSeq must come before EOStr.
  309. {"I Prefix", true}, // Reserved14-18 must come before first VCL.
  310. {"I SEI", true}, // SEI must come before first VCL.
  311. // For this case, P slice is first VCL and is detected before conformance
  312. // failure, so the non-conformant frame is reported as a non-keyframe.
  313. {"P SPS P",
  314. false}, // SPS after first VCL would indicate a new access unit.
  315. };
  316. BitstreamConverter::AnalysisResult expected;
  317. expected.is_conformant = false;
  318. for (size_t i = 0; i < std::size(test_cases); ++i) {
  319. std::vector<uint8_t> buf;
  320. std::vector<SubsampleEntry> subsamples;
  321. AvcStringToAnnexB(test_cases[i].case_string, &buf, NULL);
  322. expected.is_keyframe = test_cases[i].is_keyframe;
  323. EXPECT_PRED2(AnalysesMatch,
  324. AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
  325. expected)
  326. << "'" << test_cases[i].case_string << "' failed";
  327. }
  328. }
  329. typedef struct {
  330. const char* input;
  331. const char* expected;
  332. } InsertTestCases;
  333. TEST_F(AVCConversionTest, InsertParamSetsAnnexB) {
  334. static const InsertTestCases test_cases[] = {
  335. { "I", "SPS,SPS,PPS,I" },
  336. { "AUD I", "AUD SPS,SPS,PPS,I" },
  337. // Cases where param sets in |avc_config| are placed before
  338. // the existing ones.
  339. { "SPS,PPS,I", "SPS,SPS,PPS,SPS,PPS,I" },
  340. { "AUD,SPS,PPS,I", "AUD,SPS,SPS,PPS,SPS,PPS,I" }, // Note: params placed
  341. // after AUD.
  342. // One or more NALUs might follow AUD in the first subsample, we need to
  343. // handle this correctly. Params should be inserted right after AUD.
  344. { "AUD,SEI I", "AUD,SPS,SPS,PPS,SEI I" },
  345. };
  346. AVCDecoderConfigurationRecord avc_config;
  347. avc_config.sps_list.resize(2);
  348. avc_config.sps_list[0].push_back(0x67);
  349. avc_config.sps_list[0].push_back(0x12);
  350. avc_config.sps_list[1].push_back(0x67);
  351. avc_config.sps_list[1].push_back(0x34);
  352. avc_config.pps_list.resize(1);
  353. avc_config.pps_list[0].push_back(0x68);
  354. avc_config.pps_list[0].push_back(0x56);
  355. avc_config.pps_list[0].push_back(0x78);
  356. BitstreamConverter::AnalysisResult expected;
  357. expected.is_conformant = true;
  358. expected.is_keyframe = true;
  359. for (size_t i = 0; i < std::size(test_cases); ++i) {
  360. std::vector<uint8_t> buf;
  361. std::vector<SubsampleEntry> subsamples;
  362. AvcStringToAnnexB(test_cases[i].input, &buf, &subsamples);
  363. EXPECT_TRUE(AVC::InsertParamSetsAnnexB(avc_config, &buf, &subsamples))
  364. << "'" << test_cases[i].input << "' insert failed.";
  365. EXPECT_PRED2(AnalysesMatch,
  366. AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
  367. expected)
  368. << "'" << test_cases[i].input << "' created invalid AnnexB.";
  369. EXPECT_EQ(test_cases[i].expected, AnnexBToString(buf, subsamples))
  370. << "'" << test_cases[i].input << "' generated unexpected output.";
  371. }
  372. }
  373. } // namespace mp4
  374. } // namespace media