media_foundation_video_stream.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/renderers/win/media_foundation_video_stream.h"
  5. #include <initguid.h> // NOLINT(build/include_order)
  6. #include <mfapi.h> // NOLINT(build/include_order)
  7. #include <mferror.h> // NOLINT(build/include_order)
  8. #include <wrl.h> // NOLINT(build/include_order)
  9. #include "base/metrics/histogram_functions.h"
  10. #include "base/numerics/safe_conversions.h"
  11. #include "media/base/video_codecs.h"
  12. #include "media/base/video_decoder_config.h"
  13. #include "media/base/win/mf_helpers.h"
  14. #include "media/media_buildflags.h"
  15. namespace media {
  16. using Microsoft::WRL::ComPtr;
  17. using Microsoft::WRL::MakeAndInitialize;
  18. namespace {
  19. // This is supported by Media Foundation.
  20. DEFINE_MEDIATYPE_GUID(MFVideoFormat_THEORA, FCC('theo'))
  21. // MF_MT_MIN_MASTERING_LUMINANCE values are in 1/10000th of a nit (0.0001 nit).
  22. // https://docs.microsoft.com/en-us/windows/win32/api/dxgi1_5/ns-dxgi1_5-dxgi_hdr_metadata_hdr10
  23. constexpr int kMasteringDispLuminanceScale = 10000;
  24. GUID VideoCodecToMFSubtype(VideoCodec codec, VideoCodecProfile profile) {
  25. switch (codec) {
  26. case VideoCodec::kH264:
  27. return MFVideoFormat_H264;
  28. case VideoCodec::kVC1:
  29. return MFVideoFormat_WVC1;
  30. case VideoCodec::kMPEG2:
  31. return MFVideoFormat_MPEG2;
  32. case VideoCodec::kMPEG4:
  33. return MFVideoFormat_MP4V;
  34. case VideoCodec::kTheora:
  35. return MFVideoFormat_THEORA;
  36. case VideoCodec::kVP8:
  37. return MFVideoFormat_VP80;
  38. case VideoCodec::kVP9:
  39. return MFVideoFormat_VP90;
  40. case VideoCodec::kHEVC:
  41. return MFVideoFormat_HEVC;
  42. case VideoCodec::kDolbyVision:
  43. if (profile == VideoCodecProfile::DOLBYVISION_PROFILE0 ||
  44. profile == VideoCodecProfile::DOLBYVISION_PROFILE9) {
  45. return MFVideoFormat_H264;
  46. } else {
  47. return MFVideoFormat_HEVC;
  48. }
  49. case VideoCodec::kAV1:
  50. return MFVideoFormat_AV1;
  51. default:
  52. return GUID_NULL;
  53. }
  54. }
  55. MFVideoRotationFormat VideoRotationToMF(VideoRotation rotation) {
  56. DVLOG(2) << __func__ << ": rotation=" << rotation;
  57. switch (rotation) {
  58. case VideoRotation::VIDEO_ROTATION_0:
  59. return MFVideoRotationFormat_0;
  60. case VideoRotation::VIDEO_ROTATION_90:
  61. return MFVideoRotationFormat_90;
  62. case VideoRotation::VIDEO_ROTATION_180:
  63. return MFVideoRotationFormat_180;
  64. case VideoRotation::VIDEO_ROTATION_270:
  65. return MFVideoRotationFormat_270;
  66. }
  67. }
  68. MFVideoPrimaries VideoPrimariesToMF(
  69. media::VideoColorSpace::PrimaryID primary_id) {
  70. DVLOG(2) << __func__ << ": primary_id=" << static_cast<int>(primary_id);
  71. switch (primary_id) {
  72. case VideoColorSpace::PrimaryID::INVALID:
  73. return MFVideoPrimaries_Unknown;
  74. case VideoColorSpace::PrimaryID::BT709:
  75. return MFVideoPrimaries_BT709;
  76. case VideoColorSpace::PrimaryID::UNSPECIFIED:
  77. return MFVideoPrimaries_Unknown;
  78. case VideoColorSpace::PrimaryID::BT470M:
  79. return MFVideoPrimaries_BT470_2_SysM;
  80. case VideoColorSpace::PrimaryID::BT470BG:
  81. return MFVideoPrimaries_BT470_2_SysBG;
  82. case VideoColorSpace::PrimaryID::SMPTE170M:
  83. return MFVideoPrimaries_SMPTE170M;
  84. case VideoColorSpace::PrimaryID::SMPTE240M:
  85. return MFVideoPrimaries_SMPTE240M;
  86. case VideoColorSpace::PrimaryID::FILM:
  87. return MFVideoPrimaries_Unknown;
  88. case VideoColorSpace::PrimaryID::BT2020:
  89. return MFVideoPrimaries_BT2020;
  90. case VideoColorSpace::PrimaryID::SMPTEST428_1:
  91. return MFVideoPrimaries_Unknown;
  92. case VideoColorSpace::PrimaryID::SMPTEST431_2:
  93. return MFVideoPrimaries_Unknown;
  94. case VideoColorSpace::PrimaryID::SMPTEST432_1:
  95. return MFVideoPrimaries_Unknown;
  96. case VideoColorSpace::PrimaryID::EBU_3213_E:
  97. return MFVideoPrimaries_EBU3213;
  98. default:
  99. DLOG(ERROR) << "VideoPrimariesToMF failed due to invalid Video Primary.";
  100. }
  101. return MFVideoPrimaries_Unknown;
  102. }
  103. MFVideoTransferFunction VideoTransferFunctionToMF(
  104. media::VideoColorSpace::TransferID transfer_id) {
  105. DVLOG(2) << __func__ << ": transfer_id=" << static_cast<int>(transfer_id);
  106. switch (transfer_id) {
  107. case VideoColorSpace::TransferID::INVALID:
  108. return MFVideoTransFunc_Unknown;
  109. case VideoColorSpace::TransferID::BT709:
  110. return MFVideoTransFunc_709;
  111. case VideoColorSpace::TransferID::UNSPECIFIED:
  112. return MFVideoTransFunc_Unknown;
  113. case VideoColorSpace::TransferID::GAMMA22:
  114. return MFVideoTransFunc_22;
  115. case VideoColorSpace::TransferID::GAMMA28:
  116. return MFVideoTransFunc_28;
  117. case VideoColorSpace::TransferID::SMPTE170M:
  118. return MFVideoTransFunc_709;
  119. case VideoColorSpace::TransferID::SMPTE240M:
  120. return MFVideoTransFunc_240M;
  121. case VideoColorSpace::TransferID::LINEAR:
  122. return MFVideoTransFunc_10;
  123. case VideoColorSpace::TransferID::LOG:
  124. return MFVideoTransFunc_Log_100;
  125. case VideoColorSpace::TransferID::LOG_SQRT:
  126. return MFVideoTransFunc_Unknown;
  127. case VideoColorSpace::TransferID::IEC61966_2_4:
  128. return MFVideoTransFunc_Unknown;
  129. case VideoColorSpace::TransferID::BT1361_ECG:
  130. return MFVideoTransFunc_Unknown;
  131. case VideoColorSpace::TransferID::IEC61966_2_1:
  132. return MFVideoTransFunc_sRGB;
  133. case VideoColorSpace::TransferID::BT2020_10:
  134. return MFVideoTransFunc_2020;
  135. case VideoColorSpace::TransferID::BT2020_12:
  136. return MFVideoTransFunc_2020;
  137. case VideoColorSpace::TransferID::SMPTEST2084:
  138. return MFVideoTransFunc_2084;
  139. case VideoColorSpace::TransferID::SMPTEST428_1:
  140. return MFVideoTransFunc_Unknown;
  141. case VideoColorSpace::TransferID::ARIB_STD_B67:
  142. return MFVideoTransFunc_HLG;
  143. default:
  144. DLOG(ERROR) << "VideoTransferFunctionToMF failed due to invalid Transfer "
  145. "Function.";
  146. }
  147. return MFVideoTransFunc_Unknown;
  148. }
  149. MT_CUSTOM_VIDEO_PRIMARIES CustomVideoPrimaryToMF(
  150. gfx::ColorVolumeMetadata color_volume_metadata) {
  151. // MT_CUSTOM_VIDEO_PRIMARIES stores value in float no scaling factor needed
  152. // https://docs.microsoft.com/en-us/windows/win32/api/mfapi/ns-mfapi-mt_custom_video_primaries
  153. MT_CUSTOM_VIDEO_PRIMARIES primaries = {0};
  154. primaries.fRx = color_volume_metadata.primary_r.x();
  155. primaries.fRy = color_volume_metadata.primary_r.y();
  156. primaries.fGx = color_volume_metadata.primary_g.x();
  157. primaries.fGy = color_volume_metadata.primary_g.y();
  158. primaries.fBx = color_volume_metadata.primary_b.x();
  159. primaries.fBy = color_volume_metadata.primary_b.y();
  160. primaries.fWx = color_volume_metadata.white_point.x();
  161. primaries.fWy = color_volume_metadata.white_point.y();
  162. return primaries;
  163. }
  164. #if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  165. // To MediaFoundation, DolbyVision renderer profile strings are always 7
  166. // characters. For HEVC based profiles, it's in the format "dvhe.xx". For AVC
  167. // based profiles, it's in the format "dvav.xx". In both cases, "xx" is the
  168. // two-digit profile number. Note the DolbyVision level is ignored.
  169. bool GetDolbyVisionConfigurations(
  170. VideoCodecProfile profile,
  171. std::wstring& dolby_vision_profile,
  172. unsigned& dolby_vision_configuration_nalu_type) {
  173. DVLOG(2) << __func__ << ": profile=" << profile;
  174. switch (profile) {
  175. case VideoCodecProfile::DOLBYVISION_PROFILE0:
  176. DLOG(ERROR) << __func__ << ": Profile 0 unsupported by Media Foundation";
  177. return false;
  178. case VideoCodecProfile::DOLBYVISION_PROFILE4:
  179. dolby_vision_profile = L"dvhe.04";
  180. dolby_vision_configuration_nalu_type = 62;
  181. break;
  182. case VideoCodecProfile::DOLBYVISION_PROFILE5:
  183. dolby_vision_profile = L"dvhe.05";
  184. dolby_vision_configuration_nalu_type = 62;
  185. break;
  186. case VideoCodecProfile::DOLBYVISION_PROFILE7:
  187. dolby_vision_profile = L"dvhe.07";
  188. dolby_vision_configuration_nalu_type = 62;
  189. break;
  190. case VideoCodecProfile::DOLBYVISION_PROFILE8:
  191. dolby_vision_profile = L"dvhe.08";
  192. dolby_vision_configuration_nalu_type = 62;
  193. break;
  194. case VideoCodecProfile::DOLBYVISION_PROFILE9:
  195. dolby_vision_profile = L"dvav.09";
  196. dolby_vision_configuration_nalu_type = 28;
  197. break;
  198. default:
  199. DLOG(ERROR) << __func__ << ": Invalid profile (" << profile << ")";
  200. return false;
  201. }
  202. return true;
  203. }
  204. #endif // BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  205. // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/ns-mfobjects-mfoffset
  206. // The value of the MFOffset number is value + (fract / 65536.0f).
  207. MFOffset MakeOffset(float value) {
  208. MFOffset offset;
  209. offset.value = base::checked_cast<short>(value);
  210. offset.fract = base::checked_cast<WORD>(65536 * (value - offset.value));
  211. return offset;
  212. }
  213. // TODO(frankli): investigate if VideoCodecProfile is needed by MF.
  214. HRESULT GetVideoType(const VideoDecoderConfig& config,
  215. IMFMediaType** media_type_out) {
  216. ComPtr<IMFMediaType> media_type;
  217. RETURN_IF_FAILED(MFCreateMediaType(&media_type));
  218. RETURN_IF_FAILED(media_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
  219. GUID mf_subtype = VideoCodecToMFSubtype(config.codec(), config.profile());
  220. if (mf_subtype == GUID_NULL) {
  221. DLOG(ERROR) << "Unsupported codec type: " << config.codec();
  222. return MF_E_TOPO_CODEC_NOT_FOUND;
  223. }
  224. RETURN_IF_FAILED(media_type->SetGUID(MF_MT_SUBTYPE, mf_subtype));
  225. UINT32 width = config.visible_rect().width();
  226. UINT32 height = config.visible_rect().height();
  227. RETURN_IF_FAILED(
  228. MFSetAttributeSize(media_type.Get(), MF_MT_FRAME_SIZE, width, height));
  229. UINT32 natural_width = config.natural_size().width();
  230. UINT32 natural_height = config.natural_size().height();
  231. RETURN_IF_FAILED(
  232. MFSetAttributeRatio(media_type.Get(), MF_MT_PIXEL_ASPECT_RATIO,
  233. height * natural_width, width * natural_height));
  234. MFVideoArea area;
  235. area.OffsetX = MakeOffset(static_cast<float>(config.visible_rect().x()));
  236. area.OffsetY = MakeOffset(static_cast<float>(config.visible_rect().y()));
  237. area.Area = config.natural_size().ToSIZE();
  238. RETURN_IF_FAILED(media_type->SetBlob(MF_MT_GEOMETRIC_APERTURE, (UINT8*)&area,
  239. sizeof(area)));
  240. if (config.video_transformation().rotation !=
  241. VideoRotation::VIDEO_ROTATION_0) {
  242. MFVideoRotationFormat mf_rotation =
  243. VideoRotationToMF(config.video_transformation().rotation);
  244. RETURN_IF_FAILED(media_type->SetUINT32(MF_MT_VIDEO_ROTATION, mf_rotation));
  245. }
  246. MFVideoTransferFunction mf_transfer_function =
  247. VideoTransferFunctionToMF(config.color_space_info().transfer);
  248. RETURN_IF_FAILED(
  249. media_type->SetUINT32(MF_MT_TRANSFER_FUNCTION, mf_transfer_function));
  250. MFVideoPrimaries mf_video_primary =
  251. VideoPrimariesToMF(config.color_space_info().primaries);
  252. RETURN_IF_FAILED(
  253. media_type->SetUINT32(MF_MT_VIDEO_PRIMARIES, mf_video_primary));
  254. UINT32 video_nominal_range =
  255. config.color_space_info().range == gfx::ColorSpace::RangeID::FULL
  256. ? MFNominalRange_0_255
  257. : MFNominalRange_16_235;
  258. RETURN_IF_FAILED(
  259. media_type->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, video_nominal_range));
  260. if (config.hdr_metadata().has_value()) {
  261. UINT32 max_display_mastering_luminance =
  262. config.hdr_metadata()->color_volume_metadata.luminance_max;
  263. RETURN_IF_FAILED(media_type->SetUINT32(MF_MT_MAX_MASTERING_LUMINANCE,
  264. max_display_mastering_luminance));
  265. UINT32 min_display_mastering_luminance =
  266. config.hdr_metadata()->color_volume_metadata.luminance_min *
  267. kMasteringDispLuminanceScale;
  268. RETURN_IF_FAILED(media_type->SetUINT32(MF_MT_MIN_MASTERING_LUMINANCE,
  269. min_display_mastering_luminance));
  270. MT_CUSTOM_VIDEO_PRIMARIES primaries =
  271. CustomVideoPrimaryToMF(config.hdr_metadata()->color_volume_metadata);
  272. RETURN_IF_FAILED(media_type->SetBlob(MF_MT_CUSTOM_VIDEO_PRIMARIES,
  273. reinterpret_cast<UINT8*>(&primaries),
  274. sizeof(MT_CUSTOM_VIDEO_PRIMARIES)));
  275. }
  276. base::UmaHistogramEnumeration(
  277. "Media.MediaFoundation.VideoColorSpace.TransferID",
  278. config.color_space_info().transfer);
  279. base::UmaHistogramEnumeration(
  280. "Media.MediaFoundation.VideoColorSpace.PrimaryID",
  281. config.color_space_info().primaries);
  282. #if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  283. if (config.codec() == VideoCodec::kDolbyVision) {
  284. std::wstring dolby_vision_profile;
  285. // This value should be 28 for AVC and 62 for HEVC.
  286. unsigned dolby_vision_configuration_nalu_type = 0;
  287. if (!GetDolbyVisionConfigurations(config.profile(), dolby_vision_profile,
  288. dolby_vision_configuration_nalu_type)) {
  289. return MF_E_TOPO_CODEC_NOT_FOUND;
  290. }
  291. media_type->SetString(MF_MT_VIDEO_RENDERER_EXTENSION_PROFILE,
  292. dolby_vision_profile.c_str());
  293. media_type->SetUINT32(MF_MT_FORWARD_CUSTOM_NALU,
  294. dolby_vision_configuration_nalu_type);
  295. media_type->SetUINT32(MF_DECODER_FWD_CUSTOM_SEI_DECODE_ORDER, TRUE);
  296. }
  297. #endif // BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  298. *media_type_out = media_type.Detach();
  299. return S_OK;
  300. }
  301. } // namespace
  302. /*static*/
  303. HRESULT MediaFoundationVideoStream::Create(
  304. int stream_id,
  305. IMFMediaSource* parent_source,
  306. DemuxerStream* demuxer_stream,
  307. std::unique_ptr<MediaLog> media_log,
  308. MediaFoundationStreamWrapper** stream_out) {
  309. DVLOG(1) << __func__ << ": stream_id=" << stream_id;
  310. ComPtr<IMFMediaStream> video_stream;
  311. VideoCodec codec = demuxer_stream->video_decoder_config().codec();
  312. switch (codec) {
  313. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  314. case VideoCodec::kH264:
  315. RETURN_IF_FAILED(MakeAndInitialize<MediaFoundationH264VideoStream>(
  316. &video_stream, stream_id, parent_source, demuxer_stream,
  317. std::move(media_log)));
  318. break;
  319. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  320. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  321. case VideoCodec::kHEVC:
  322. #endif
  323. #if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  324. case VideoCodec::kDolbyVision:
  325. #endif
  326. #if BUILDFLAG(ENABLE_PLATFORM_HEVC) || BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  327. RETURN_IF_FAILED(MakeAndInitialize<MediaFoundationHEVCVideoStream>(
  328. &video_stream, stream_id, parent_source, demuxer_stream,
  329. std::move(media_log)));
  330. break;
  331. #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) ||
  332. // BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
  333. default:
  334. RETURN_IF_FAILED(MakeAndInitialize<MediaFoundationVideoStream>(
  335. &video_stream, stream_id, parent_source, demuxer_stream,
  336. std::move(media_log)));
  337. break;
  338. }
  339. *stream_out =
  340. static_cast<MediaFoundationStreamWrapper*>(video_stream.Detach());
  341. return S_OK;
  342. }
  343. bool MediaFoundationVideoStream::IsEncrypted() const {
  344. VideoDecoderConfig decoder_config = demuxer_stream_->video_decoder_config();
  345. return decoder_config.is_encrypted();
  346. }
  347. HRESULT MediaFoundationVideoStream::GetMediaType(
  348. IMFMediaType** media_type_out) {
  349. VideoDecoderConfig decoder_config = demuxer_stream_->video_decoder_config();
  350. return GetVideoType(decoder_config, media_type_out);
  351. }
  352. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  353. HRESULT MediaFoundationH264VideoStream::GetMediaType(
  354. IMFMediaType** media_type_out) {
  355. VideoDecoderConfig decoder_config = demuxer_stream_->video_decoder_config();
  356. RETURN_IF_FAILED(GetVideoType(decoder_config, media_type_out));
  357. // Enable conversion to Annex-B
  358. demuxer_stream_->EnableBitstreamConverter();
  359. return S_OK;
  360. }
  361. bool MediaFoundationH264VideoStream::AreFormatChangesEnabled() {
  362. // Disable explicit format change event for H264 to allow switching to the
  363. // new stream without a full re-create, which will be much faster. This is
  364. // also due to the fact that the MFT decoder can handle some format changes
  365. // without a format change event. For format changes that the MFT decoder
  366. // cannot support (e.g. codec change), the playback will fail later with
  367. // MF_E_INVALIDMEDIATYPE (0xC00D36B4).
  368. return false;
  369. }
  370. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  371. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  372. bool MediaFoundationHEVCVideoStream::AreFormatChangesEnabled() {
  373. // Disable explicit format change event for HEVC to allow switching to the
  374. // new stream without a full re-create, which will be much faster. This is
  375. // also due to the fact that the MFT decoder can handle some format changes
  376. // without a format change event. For format changes that the MFT decoder
  377. // cannot support (e.g. codec change), the playback will fail later with
  378. // MF_E_INVALIDMEDIATYPE (0xC00D36B4).
  379. return false;
  380. }
  381. #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC)
  382. } // namespace media