vt_video_decode_accelerator_mac.cc 89 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/mac/vt_video_decode_accelerator_mac.h"
  5. #include <CoreFoundation/CoreFoundation.h>
  6. #include <CoreVideo/CoreVideo.h>
  7. #include <OpenGL/CGLIOSurface.h>
  8. #include <OpenGL/gl.h>
  9. #include <stddef.h>
  10. #include <algorithm>
  11. #include <iterator>
  12. #include <memory>
  13. #include "base/atomic_sequence_num.h"
  14. #include "base/bind.h"
  15. #include "base/cxx17_backports.h"
  16. #include "base/logging.h"
  17. #include "base/mac/mac_logging.h"
  18. #include "base/mac/mac_util.h"
  19. #include "base/mac/sdk_forward_declarations.h"
  20. #include "base/memory/ptr_util.h"
  21. #include "base/metrics/histogram_macros.h"
  22. #include "base/no_destructor.h"
  23. #include "base/numerics/safe_conversions.h"
  24. #include "base/stl_util.h"
  25. #include "base/strings/stringprintf.h"
  26. #include "base/strings/sys_string_conversions.h"
  27. #include "base/sys_byteorder.h"
  28. #include "base/system/sys_info.h"
  29. #include "base/task/task_traits.h"
  30. #include "base/task/thread_pool.h"
  31. #include "base/threading/thread_task_runner_handle.h"
  32. #include "base/trace_event/memory_allocator_dump.h"
  33. #include "base/trace_event/memory_dump_manager.h"
  34. #include "base/trace_event/process_memory_dump.h"
  35. #include "base/version.h"
  36. #include "components/crash/core/common/crash_key.h"
  37. #include "components/viz/common/resources/resource_format_utils.h"
  38. #include "gpu/command_buffer/common/gpu_memory_buffer_support.h"
  39. #include "gpu/command_buffer/common/mailbox.h"
  40. #include "gpu/command_buffer/common/shared_image_usage.h"
  41. #include "gpu/command_buffer/service/shared_image/gl_image_backing.h"
  42. #include "gpu/command_buffer/service/shared_image/shared_image_factory.h"
  43. #include "gpu/ipc/service/shared_image_stub.h"
  44. #include "media/base/limits.h"
  45. #include "media/base/mac/color_space_util_mac.h"
  46. #include "media/base/media_switches.h"
  47. #include "media/filters/vp9_parser.h"
  48. #include "media/gpu/mac/vp9_super_frame_bitstream_filter.h"
  49. #include "media/gpu/mac/vt_config_util.h"
  50. #include "media/video/h264_level_limits.h"
  51. #include "ui/gfx/geometry/rect.h"
  52. #include "ui/gl/gl_context.h"
  53. #include "ui/gl/gl_image_io_surface.h"
  54. #include "ui/gl/gl_implementation.h"
  55. #include "ui/gl/scoped_binders.h"
  56. #define NOTIFY_STATUS(name, status, session_failure) \
  57. do { \
  58. OSSTATUS_DLOG(ERROR, status) << name; \
  59. NotifyError(PLATFORM_FAILURE, session_failure); \
  60. } while (0)
  61. namespace media {
  62. namespace {
  63. // A sequence of ids for memory tracing.
  64. base::AtomicSequenceNumber g_memory_dump_ids;
  65. // A sequence of shared memory ids for CVPixelBufferRefs.
  66. base::AtomicSequenceNumber g_cv_pixel_buffer_ids;
  67. // Only H.264 with 4:2:0 chroma sampling is supported.
  68. constexpr VideoCodecProfile kSupportedProfiles[] = {
  69. H264PROFILE_BASELINE, H264PROFILE_EXTENDED, H264PROFILE_MAIN,
  70. H264PROFILE_HIGH,
  71. // These are only supported on macOS 11+.
  72. VP9PROFILE_PROFILE0, VP9PROFILE_PROFILE2,
  73. // These are only supported on macOS 11+.
  74. HEVCPROFILE_MAIN, HEVCPROFILE_MAIN10, HEVCPROFILE_MAIN_STILL_PICTURE,
  75. // This is partially supported on macOS 11+, Apple Silicon Mac only supports
  76. // 8 ~ 10 bit 400, 420, 422, 444 HW decoding, and Intel Mac supports 8 ~ 12
  77. // bit 400, 420, 422 SW decoding, 444 content is decodable but has a green
  78. // stripe issue.
  79. HEVCPROFILE_REXT,
  80. // TODO(sandersd): Hi10p fails during
  81. // CMVideoFormatDescriptionCreateFromH264ParameterSets with
  82. // kCMFormatDescriptionError_InvalidParameter.
  83. //
  84. // H264PROFILE_HIGH10PROFILE,
  85. // TODO(sandersd): Find and test media with these profiles before enabling.
  86. //
  87. // H264PROFILE_SCALABLEBASELINE,
  88. // H264PROFILE_SCALABLEHIGH,
  89. // H264PROFILE_STEREOHIGH,
  90. // H264PROFILE_MULTIVIEWHIGH,
  91. };
  92. // Size to use for NALU length headers in AVC format (can be 1, 2, or 4).
  93. constexpr int kNALUHeaderLength = 4;
  94. // We request 16 picture buffers from the client, each of which has a texture ID
  95. // that we can bind decoded frames to. The resource requirements are low, as we
  96. // don't need the textures to be backed by storage.
  97. //
  98. // The lower limit is |limits::kMaxVideoFrames + 1|, enough to have one
  99. // composited frame plus |limits::kMaxVideoFrames| frames to satisfy preroll.
  100. //
  101. // However, there can be pathological behavior where VideoRendererImpl will
  102. // continue to call Decode() as long as it is willing to queue more output
  103. // frames, which is variable but starts at |limits::kMaxVideoFrames +
  104. // GetMaxDecodeRequests()|. If we don't have enough picture buffers, it will
  105. // continue to call Decode() until we stop calling NotifyEndOfBistreamBuffer(),
  106. // which for VTVDA is when the reorder queue is full. In testing this results in
  107. // ~20 extra frames held by VTVDA.
  108. //
  109. // Allocating more picture buffers than VideoRendererImpl is willing to queue
  110. // counterintuitively reduces memory usage in this case.
  111. constexpr int kNumPictureBuffers = limits::kMaxVideoFrames * 4;
  112. // Maximum number of frames to queue for reordering. (Also controls the maximum
  113. // number of in-flight frames, since NotifyEndOfBitstreamBuffer() is called when
  114. // frames are moved into the reorder queue.)
  115. //
  116. // Since the maximum possible |reorder_window| is 16 for H.264, 17 is the
  117. // minimum safe (static) size of the reorder queue.
  118. constexpr int kMaxReorderQueueSize = 17;
  119. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  120. // If videotoolbox total output picture count is lower than
  121. // kMinOutputsBeforeRASL, then we should skip the RASL frames
  122. // to avoid kVTVideoDecoderBadDataErr
  123. constexpr int kMinOutputsBeforeRASL = 5;
  124. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  125. // Build an |image_config| dictionary for VideoToolbox initialization.
  126. base::ScopedCFTypeRef<CFMutableDictionaryRef> BuildImageConfig(
  127. CMVideoDimensions coded_dimensions,
  128. bool is_hbd) {
  129. base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config;
  130. // Note that 4:2:0 textures cannot be used directly as RGBA in OpenGL, but are
  131. // lower power than 4:2:2 when composited directly by CoreAnimation.
  132. int32_t pixel_format = is_hbd
  133. ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
  134. : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
  135. #define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
  136. base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
  137. base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
  138. base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
  139. #undef CFINT
  140. if (!cf_pixel_format.get() || !cf_width.get() || !cf_height.get())
  141. return image_config;
  142. image_config.reset(CFDictionaryCreateMutable(
  143. kCFAllocatorDefault,
  144. 3, // capacity
  145. &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
  146. if (!image_config.get())
  147. return image_config;
  148. CFDictionarySetValue(image_config, kCVPixelBufferPixelFormatTypeKey,
  149. cf_pixel_format);
  150. CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
  151. CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
  152. return image_config;
  153. }
  154. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  155. // Create a CMFormatDescription using the provided |pps|, |sps| and |vps|.
  156. base::ScopedCFTypeRef<CMFormatDescriptionRef> CreateVideoFormatHEVC(
  157. const std::vector<uint8_t>& vps,
  158. const std::vector<uint8_t>& sps,
  159. const std::vector<uint8_t>& pps) {
  160. DCHECK(!vps.empty());
  161. DCHECK(!sps.empty());
  162. DCHECK(!pps.empty());
  163. // Build the configuration records.
  164. std::vector<const uint8_t*> nalu_data_ptrs;
  165. std::vector<size_t> nalu_data_sizes;
  166. nalu_data_ptrs.reserve(3);
  167. nalu_data_sizes.reserve(3);
  168. nalu_data_ptrs.push_back(&vps.front());
  169. nalu_data_sizes.push_back(vps.size());
  170. nalu_data_ptrs.push_back(&sps.front());
  171. nalu_data_sizes.push_back(sps.size());
  172. nalu_data_ptrs.push_back(&pps.front());
  173. nalu_data_sizes.push_back(pps.size());
  174. // For some unknown reason, even if apple has claimed that this API is
  175. // available after macOS 10.13, however base on the result on macOS 10.15.7,
  176. // we could get an OSStatus=-12906 kVTCouldNotFindVideoDecoderErr after
  177. // calling VTDecompressionSessionCreate(), so macOS 11+ is necessary
  178. // (https://crbug.com/1300444#c9)
  179. base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
  180. if (__builtin_available(macOS 11.0, *)) {
  181. OSStatus status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
  182. kCFAllocatorDefault,
  183. nalu_data_ptrs.size(), // parameter_set_count
  184. &nalu_data_ptrs.front(), // &parameter_set_pointers
  185. &nalu_data_sizes.front(), // &parameter_set_sizes
  186. kNALUHeaderLength, // nal_unit_header_length
  187. NULL, format.InitializeInto());
  188. OSSTATUS_DLOG_IF(WARNING, status != noErr, status)
  189. << "CMVideoFormatDescriptionCreateFromHEVCParameterSets()";
  190. }
  191. return format;
  192. }
  193. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  194. // Create a CMFormatDescription using the provided |pps| and |sps|.
  195. base::ScopedCFTypeRef<CMFormatDescriptionRef> CreateVideoFormatH264(
  196. const std::vector<uint8_t>& sps,
  197. const std::vector<uint8_t>& spsext,
  198. const std::vector<uint8_t>& pps) {
  199. DCHECK(!sps.empty());
  200. DCHECK(!pps.empty());
  201. // Build the configuration records.
  202. std::vector<const uint8_t*> nalu_data_ptrs;
  203. std::vector<size_t> nalu_data_sizes;
  204. nalu_data_ptrs.reserve(3);
  205. nalu_data_sizes.reserve(3);
  206. nalu_data_ptrs.push_back(&sps.front());
  207. nalu_data_sizes.push_back(sps.size());
  208. if (!spsext.empty()) {
  209. nalu_data_ptrs.push_back(&spsext.front());
  210. nalu_data_sizes.push_back(spsext.size());
  211. }
  212. nalu_data_ptrs.push_back(&pps.front());
  213. nalu_data_sizes.push_back(pps.size());
  214. // Construct a new format description from the parameter sets.
  215. base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
  216. OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
  217. kCFAllocatorDefault,
  218. nalu_data_ptrs.size(), // parameter_set_count
  219. &nalu_data_ptrs.front(), // &parameter_set_pointers
  220. &nalu_data_sizes.front(), // &parameter_set_sizes
  221. kNALUHeaderLength, // nal_unit_header_length
  222. format.InitializeInto());
  223. OSSTATUS_DLOG_IF(WARNING, status != noErr, status)
  224. << "CMVideoFormatDescriptionCreateFromH264ParameterSets()";
  225. return format;
  226. }
  227. base::ScopedCFTypeRef<CMFormatDescriptionRef> CreateVideoFormatVP9(
  228. media::VideoColorSpace color_space,
  229. media::VideoCodecProfile profile,
  230. absl::optional<gfx::HDRMetadata> hdr_metadata,
  231. const gfx::Size& coded_size) {
  232. base::ScopedCFTypeRef<CFMutableDictionaryRef> format_config(
  233. CreateFormatExtensions(kCMVideoCodecType_VP9, profile, color_space,
  234. hdr_metadata));
  235. base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
  236. if (!format_config) {
  237. DLOG(ERROR) << "Failed to configure vp9 decoder.";
  238. return format;
  239. }
  240. OSStatus status = CMVideoFormatDescriptionCreate(
  241. kCFAllocatorDefault, kCMVideoCodecType_VP9, coded_size.width(),
  242. coded_size.height(), format_config, format.InitializeInto());
  243. OSSTATUS_DLOG_IF(WARNING, status != noErr, status)
  244. << "CMVideoFormatDescriptionCreate()";
  245. return format;
  246. }
  247. // Create a VTDecompressionSession using the provided |format|. If
  248. // |require_hardware| is true, the session will only use the hardware decoder.
  249. bool CreateVideoToolboxSession(
  250. const CMFormatDescriptionRef format,
  251. bool require_hardware,
  252. bool is_hbd,
  253. const VTDecompressionOutputCallbackRecord* callback,
  254. base::ScopedCFTypeRef<VTDecompressionSessionRef>* session,
  255. gfx::Size* configured_size) {
  256. // Prepare VideoToolbox configuration dictionaries.
  257. base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
  258. CFDictionaryCreateMutable(kCFAllocatorDefault,
  259. 1, // capacity
  260. &kCFTypeDictionaryKeyCallBacks,
  261. &kCFTypeDictionaryValueCallBacks));
  262. if (!decoder_config) {
  263. DLOG(ERROR) << "Failed to create CFMutableDictionary";
  264. return false;
  265. }
  266. CFDictionarySetValue(
  267. decoder_config,
  268. kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
  269. kCFBooleanTrue);
  270. CFDictionarySetValue(
  271. decoder_config,
  272. kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
  273. require_hardware ? kCFBooleanTrue : kCFBooleanFalse);
  274. // VideoToolbox scales the visible rect to the output size, so we set the
  275. // output size for a 1:1 ratio. (Note though that VideoToolbox does not handle
  276. // top or left crops correctly.) We expect the visible rect to be integral.
  277. CGRect visible_rect = CMVideoFormatDescriptionGetCleanAperture(format, true);
  278. CMVideoDimensions visible_dimensions = {
  279. base::ClampFloor(visible_rect.size.width),
  280. base::ClampFloor(visible_rect.size.height)};
  281. base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
  282. BuildImageConfig(visible_dimensions, is_hbd));
  283. if (!image_config) {
  284. DLOG(ERROR) << "Failed to create decoder image configuration";
  285. return false;
  286. }
  287. OSStatus status = VTDecompressionSessionCreate(
  288. kCFAllocatorDefault,
  289. format, // video_format_description
  290. decoder_config, // video_decoder_specification
  291. image_config, // destination_image_buffer_attributes
  292. callback, // output_callback
  293. session->InitializeInto());
  294. if (status != noErr) {
  295. OSSTATUS_DLOG(WARNING, status) << "VTDecompressionSessionCreate()";
  296. return false;
  297. }
  298. *configured_size =
  299. gfx::Size(visible_rect.size.width, visible_rect.size.height);
  300. return true;
  301. }
  302. // The purpose of this function is to preload the generic and hardware-specific
  303. // libraries required by VideoToolbox before the GPU sandbox is enabled.
  304. // VideoToolbox normally loads the hardware-specific libraries lazily, so we
  305. // must actually create a decompression session. If creating a decompression
  306. // session fails, hardware decoding will be disabled (Initialize() will always
  307. // return false).
  308. bool InitializeVideoToolboxInternal() {
  309. VTDecompressionOutputCallbackRecord callback = {0};
  310. base::ScopedCFTypeRef<VTDecompressionSessionRef> session;
  311. gfx::Size configured_size;
  312. // Create a h264 hardware decoding session.
  313. // SPS and PPS data are taken from a 480p sample (buck2.mp4).
  314. const std::vector<uint8_t> sps_h264_normal = {
  315. 0x67, 0x64, 0x00, 0x1e, 0xac, 0xd9, 0x80, 0xd4, 0x3d, 0xa1, 0x00, 0x00,
  316. 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, 0x30, 0x8f, 0x16, 0x2d, 0x9a};
  317. const std::vector<uint8_t> pps_h264_normal = {0x68, 0xe9, 0x7b, 0xcb};
  318. if (!CreateVideoToolboxSession(
  319. CreateVideoFormatH264(sps_h264_normal, std::vector<uint8_t>(),
  320. pps_h264_normal),
  321. /*require_hardware=*/true, /*is_hbd=*/false, &callback, &session,
  322. &configured_size)) {
  323. DVLOG(1) << "Hardware H264 decoding with VideoToolbox is not supported";
  324. return false;
  325. }
  326. session.reset();
  327. // Create a h264 software decoding session.
  328. // SPS and PPS data are taken from a 18p sample (small2.mp4).
  329. const std::vector<uint8_t> sps_h264_small = {
  330. 0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x89, 0x7e, 0x22, 0x10, 0x00,
  331. 0x00, 0x3e, 0x90, 0x00, 0x0e, 0xa6, 0x08, 0xf1, 0x22, 0x59, 0xa0};
  332. const std::vector<uint8_t> pps_h264_small = {0x68, 0xe9, 0x79, 0x72, 0xc0};
  333. if (!CreateVideoToolboxSession(
  334. CreateVideoFormatH264(sps_h264_small, std::vector<uint8_t>(),
  335. pps_h264_small),
  336. /*require_hardware=*/false, /*is_hbd=*/false, &callback, &session,
  337. &configured_size)) {
  338. DVLOG(1) << "Software H264 decoding with VideoToolbox is not supported";
  339. return false;
  340. }
  341. session.reset();
  342. if (__builtin_available(macOS 11.0, *)) {
  343. VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9);
  344. // Create a VP9 decoding session.
  345. if (!CreateVideoToolboxSession(
  346. CreateVideoFormatVP9(VideoColorSpace::REC709(), VP9PROFILE_PROFILE0,
  347. absl::nullopt, gfx::Size(720, 480)),
  348. /*require_hardware=*/true, /*is_hbd=*/false, &callback, &session,
  349. &configured_size)) {
  350. DVLOG(1) << "Hardware VP9 decoding with VideoToolbox is not supported";
  351. // We don't return false here since VP9 support is optional.
  352. }
  353. }
  354. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  355. if (base::FeatureList::IsEnabled(media::kPlatformHEVCDecoderSupport)) {
  356. // Only macOS >= 11.0 will support hevc if we use
  357. // CMVideoFormatDescriptionCreateFromHEVCParameterSets
  358. // API to create video format
  359. if (__builtin_available(macOS 11.0, *)) {
  360. session.reset();
  361. // Create a hevc hardware decoding session.
  362. // VPS, SPS and PPS data are taken from a 720p sample
  363. // (bear-1280x720-hevc.mp4).
  364. const std::vector<uint8_t> vps_hevc_normal = {
  365. 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60,
  366. 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03,
  367. 0x00, 0x00, 0x03, 0x00, 0x5d, 0x95, 0x98, 0x09};
  368. const std::vector<uint8_t> sps_hevc_normal = {
  369. 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00,
  370. 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5d, 0xa0, 0x02, 0x80, 0x80,
  371. 0x2d, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x5a, 0x70, 0x80,
  372. 0x00, 0x01, 0xf4, 0x80, 0x00, 0x3a, 0x98, 0x04};
  373. const std::vector<uint8_t> pps_hevc_normal = {0x44, 0x01, 0xc1, 0x72,
  374. 0xb4, 0x62, 0x40};
  375. if (!CreateVideoToolboxSession(
  376. CreateVideoFormatHEVC(vps_hevc_normal, sps_hevc_normal,
  377. pps_hevc_normal),
  378. /*require_hardware=*/true, /*is_hbd=*/false, &callback, &session,
  379. &configured_size)) {
  380. DVLOG(1) << "Hardware HEVC decoding with VideoToolbox is not supported";
  381. // We don't return false here since HEVC support is optional.
  382. }
  383. session.reset();
  384. // Create a hevc software decoding session.
  385. // VPS, SPS and PPS data are taken from a 240p sample
  386. // (bear-320x240-v_frag-hevc.mp4).
  387. const std::vector<uint8_t> vps_hevc_small = {
  388. 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60,
  389. 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03,
  390. 0x00, 0x00, 0x03, 0x00, 0x3c, 0x95, 0x98, 0x09};
  391. const std::vector<uint8_t> sps_hevc_small = {
  392. 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90,
  393. 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x3c, 0xa0, 0x0a,
  394. 0x08, 0x0f, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x40,
  395. 0x40, 0x00, 0x00, 0xfa, 0x40, 0x00, 0x1d, 0x4c, 0x02};
  396. const std::vector<uint8_t> pps_hevc_small = {0x44, 0x01, 0xc1, 0x72,
  397. 0xb4, 0x62, 0x40};
  398. if (!CreateVideoToolboxSession(
  399. CreateVideoFormatHEVC(vps_hevc_small, sps_hevc_small,
  400. pps_hevc_small),
  401. /*require_hardware=*/false, /*is_hbd=*/false, &callback, &session,
  402. &configured_size)) {
  403. DVLOG(1) << "Software HEVC decoding with VideoToolbox is not supported";
  404. // We don't return false here since HEVC support is optional.
  405. }
  406. }
  407. }
  408. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  409. return true;
  410. }
  411. // TODO(sandersd): Share this computation with the VAAPI decoder.
  412. int32_t ComputeH264ReorderWindow(const H264SPS* sps) {
  413. // When |pic_order_cnt_type| == 2, decode order always matches presentation
  414. // order.
  415. // TODO(sandersd): For |pic_order_cnt_type| == 1, analyze the delta cycle to
  416. // find the minimum required reorder window.
  417. if (sps->pic_order_cnt_type == 2)
  418. return 0;
  419. int max_dpb_mbs = H264LevelToMaxDpbMbs(sps->GetIndicatedLevel());
  420. int max_dpb_frames =
  421. max_dpb_mbs / ((sps->pic_width_in_mbs_minus1 + 1) *
  422. (sps->pic_height_in_map_units_minus1 + 1));
  423. max_dpb_frames = base::clamp(max_dpb_frames, 0, 16);
  424. // See AVC spec section E.2.1 definition of |max_num_reorder_frames|.
  425. if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
  426. return std::min(sps->max_num_reorder_frames, max_dpb_frames);
  427. } else if (sps->constraint_set3_flag) {
  428. if (sps->profile_idc == 44 || sps->profile_idc == 86 ||
  429. sps->profile_idc == 100 || sps->profile_idc == 110 ||
  430. sps->profile_idc == 122 || sps->profile_idc == 244) {
  431. return 0;
  432. }
  433. }
  434. return max_dpb_frames;
  435. }
  436. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  437. int32_t ComputeHEVCReorderWindow(const H265VPS* vps) {
  438. int32_t vps_max_sub_layers_minus1 = vps->vps_max_sub_layers_minus1;
  439. return vps->vps_max_num_reorder_pics[vps_max_sub_layers_minus1];
  440. }
  441. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  442. // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
  443. void OutputThunk(void* decompression_output_refcon,
  444. void* source_frame_refcon,
  445. OSStatus status,
  446. VTDecodeInfoFlags info_flags,
  447. CVImageBufferRef image_buffer,
  448. CMTime presentation_time_stamp,
  449. CMTime presentation_duration) {
  450. VTVideoDecodeAccelerator* vda =
  451. reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
  452. vda->Output(source_frame_refcon, status, image_buffer);
  453. }
  454. } // namespace
  455. // Detects coded size and color space changes. Also indicates when a frame won't
  456. // generate any output.
  457. class VP9ConfigChangeDetector {
  458. public:
  459. VP9ConfigChangeDetector() : vp9_parser_(false) {}
  460. ~VP9ConfigChangeDetector() = default;
  461. void DetectConfig(const uint8_t* stream, unsigned int size) {
  462. vp9_parser_.SetStream(stream, size, nullptr);
  463. config_changed_ = false;
  464. Vp9FrameHeader fhdr;
  465. gfx::Size allocate_size;
  466. std::unique_ptr<DecryptConfig> null_config;
  467. while (vp9_parser_.ParseNextFrame(&fhdr, &allocate_size, &null_config) ==
  468. Vp9Parser::kOk) {
  469. color_space_ = fhdr.GetColorSpace();
  470. gfx::Size new_size(fhdr.frame_width, fhdr.frame_height);
  471. if (!size_.IsEmpty() && !pending_config_changed_ && !config_changed_ &&
  472. size_ != new_size) {
  473. pending_config_changed_ = true;
  474. DVLOG(1) << "Configuration changed from " << size_.ToString() << " to "
  475. << new_size.ToString();
  476. }
  477. size_ = new_size;
  478. // Resolution changes can happen on any frame technically, so wait for a
  479. // keyframe before signaling the config change.
  480. if (fhdr.IsKeyframe() && pending_config_changed_) {
  481. config_changed_ = true;
  482. pending_config_changed_ = false;
  483. }
  484. }
  485. if (pending_config_changed_)
  486. DVLOG(1) << "Deferring config change until next keyframe...";
  487. }
  488. gfx::Size GetCodedSize(const gfx::Size& container_coded_size) const {
  489. return size_.IsEmpty() ? container_coded_size : size_;
  490. }
  491. VideoColorSpace GetColorSpace(const VideoColorSpace& container_cs) const {
  492. return container_cs.IsSpecified() ? container_cs : color_space_;
  493. }
  494. bool config_changed() const { return config_changed_; }
  495. private:
  496. gfx::Size size_;
  497. bool config_changed_ = false;
  498. bool pending_config_changed_ = false;
  499. VideoColorSpace color_space_;
  500. Vp9Parser vp9_parser_;
  501. };
  502. bool InitializeVideoToolbox() {
  503. // InitializeVideoToolbox() is called only from the GPU process main thread:
  504. // once for sandbox warmup, and then once each time a VTVideoDecodeAccelerator
  505. // is initialized. This ensures that everything is loaded whether or not the
  506. // sandbox is enabled.
  507. static const bool succeeded = InitializeVideoToolboxInternal();
  508. return succeeded;
  509. }
  510. VTVideoDecodeAccelerator::Task::Task(TaskType type) : type(type) {}
  511. VTVideoDecodeAccelerator::Task::Task(Task&& other) = default;
  512. VTVideoDecodeAccelerator::Task::~Task() {}
  513. VTVideoDecodeAccelerator::Frame::Frame(int32_t bitstream_id)
  514. : bitstream_id(bitstream_id) {}
  515. VTVideoDecodeAccelerator::Frame::~Frame() {}
  516. VTVideoDecodeAccelerator::PictureInfo::PictureInfo()
  517. : uses_shared_images(true) {}
  518. VTVideoDecodeAccelerator::PictureInfo::PictureInfo(uint32_t client_texture_id,
  519. uint32_t service_texture_id)
  520. : uses_shared_images(false),
  521. client_texture_id(client_texture_id),
  522. service_texture_id(service_texture_id) {}
  523. VTVideoDecodeAccelerator::PictureInfo::~PictureInfo() {}
  524. bool VTVideoDecodeAccelerator::FrameOrder::operator()(
  525. const std::unique_ptr<Frame>& lhs,
  526. const std::unique_ptr<Frame>& rhs) const {
  527. // TODO(sandersd): When it is provided, use the bitstream timestamp.
  528. if (lhs->pic_order_cnt != rhs->pic_order_cnt)
  529. return lhs->pic_order_cnt > rhs->pic_order_cnt;
  530. // If |pic_order_cnt| is the same, fall back on using the bitstream order.
  531. return lhs->bitstream_id > rhs->bitstream_id;
  532. }
  533. VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(
  534. const GpuVideoDecodeGLClient& gl_client,
  535. const gpu::GpuDriverBugWorkarounds& workarounds,
  536. MediaLog* media_log)
  537. : gl_client_(gl_client),
  538. workarounds_(workarounds),
  539. // Non media/ use cases like PPAPI may not provide a MediaLog.
  540. media_log_(media_log ? media_log->Clone() : nullptr),
  541. gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  542. decoder_task_runner_(base::ThreadPool::CreateSequencedTaskRunner(
  543. {base::TaskPriority::USER_VISIBLE})),
  544. decoder_weak_this_factory_(this),
  545. weak_this_factory_(this) {
  546. DCHECK(gl_client_.bind_image);
  547. callback_.decompressionOutputCallback = OutputThunk;
  548. callback_.decompressionOutputRefCon = this;
  549. decoder_weak_this_ = decoder_weak_this_factory_.GetWeakPtr();
  550. weak_this_ = weak_this_factory_.GetWeakPtr();
  551. memory_dump_id_ = g_memory_dump_ids.GetNext();
  552. base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
  553. this, "VTVideoDecodeAccelerator", gpu_task_runner_);
  554. }
  555. VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
  556. DVLOG(1) << __func__;
  557. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  558. base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
  559. this);
  560. }
  561. bool VTVideoDecodeAccelerator::OnMemoryDump(
  562. const base::trace_event::MemoryDumpArgs& args,
  563. base::trace_event::ProcessMemoryDump* pmd) {
  564. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  565. // Dump output pictures (decoded frames for which PictureReady() has been
  566. // called already).
  567. for (const auto& [texture_id, picture_info] : picture_info_map_) {
  568. for (const auto& gl_image : picture_info->gl_images) {
  569. std::string dump_name =
  570. base::StringPrintf("media/vt_video_decode_accelerator_%d/picture_%d",
  571. memory_dump_id_, picture_info->bitstream_id);
  572. gl_image->OnMemoryDump(pmd, 0, dump_name);
  573. }
  574. }
  575. // Dump the output queue (decoded frames for which
  576. // NotifyEndOfBitstreamBuffer() has not been called yet).
  577. {
  578. uint64_t total_count = 0;
  579. uint64_t total_size = 0;
  580. for (const auto& it : base::GetUnderlyingContainer(task_queue_)) {
  581. if (it.frame.get() && it.frame->image) {
  582. IOSurfaceRef io_surface = CVPixelBufferGetIOSurface(it.frame->image);
  583. if (io_surface) {
  584. ++total_count;
  585. total_size += IOSurfaceGetAllocSize(io_surface);
  586. }
  587. }
  588. }
  589. base::trace_event::MemoryAllocatorDump* dump = pmd->CreateAllocatorDump(
  590. base::StringPrintf("media/vt_video_decode_accelerator_%d/output_queue",
  591. memory_dump_id_));
  592. dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameObjectCount,
  593. base::trace_event::MemoryAllocatorDump::kUnitsObjects,
  594. total_count);
  595. dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
  596. base::trace_event::MemoryAllocatorDump::kUnitsBytes,
  597. total_size);
  598. }
  599. // Dump the reorder queue (decoded frames for which
  600. // NotifyEndOfBitstreamBuffer() has been called already).
  601. {
  602. uint64_t total_count = 0;
  603. uint64_t total_size = 0;
  604. for (const auto& it : base::GetUnderlyingContainer(reorder_queue_)) {
  605. if (it.get() && it->image) {
  606. IOSurfaceRef io_surface = CVPixelBufferGetIOSurface(it->image);
  607. if (io_surface) {
  608. ++total_count;
  609. total_size += IOSurfaceGetAllocSize(io_surface);
  610. }
  611. }
  612. }
  613. base::trace_event::MemoryAllocatorDump* dump = pmd->CreateAllocatorDump(
  614. base::StringPrintf("media/vt_video_decode_accelerator_%d/reorder_queue",
  615. memory_dump_id_));
  616. dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameObjectCount,
  617. base::trace_event::MemoryAllocatorDump::kUnitsObjects,
  618. total_count);
  619. dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
  620. base::trace_event::MemoryAllocatorDump::kUnitsBytes,
  621. total_size);
  622. }
  623. return true;
  624. }
  625. bool VTVideoDecodeAccelerator::Initialize(const Config& config,
  626. Client* client) {
  627. DVLOG(1) << __func__;
  628. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  629. // All of these checks should be handled by the caller inspecting
  630. // SupportedProfiles(). PPAPI does not do that, however.
  631. if (config.output_mode != Config::OutputMode::ALLOCATE) {
  632. DVLOG(2) << "Output mode must be ALLOCATE";
  633. return false;
  634. }
  635. if (config.is_encrypted()) {
  636. DVLOG(2) << "Encrypted streams are not supported";
  637. return false;
  638. }
  639. static const base::NoDestructor<VideoDecodeAccelerator::SupportedProfiles>
  640. kActualSupportedProfiles(GetSupportedProfiles(workarounds_));
  641. if (std::find_if(kActualSupportedProfiles->begin(),
  642. kActualSupportedProfiles->end(), [config](const auto& p) {
  643. return p.profile == config.profile;
  644. }) == kActualSupportedProfiles->end()) {
  645. DVLOG(2) << "Unsupported profile";
  646. return false;
  647. }
  648. if (!InitializeVideoToolbox()) {
  649. DVLOG(2) << "VideoToolbox is unavailable";
  650. return false;
  651. }
  652. client_ = client;
  653. config_ = config;
  654. switch (config.profile) {
  655. case H264PROFILE_BASELINE:
  656. case H264PROFILE_EXTENDED:
  657. case H264PROFILE_MAIN:
  658. case H264PROFILE_HIGH:
  659. codec_ = VideoCodec::kH264;
  660. break;
  661. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  662. case HEVCPROFILE_MAIN:
  663. case HEVCPROFILE_MAIN10:
  664. case HEVCPROFILE_MAIN_STILL_PICTURE:
  665. case HEVCPROFILE_REXT:
  666. codec_ = VideoCodec::kHEVC;
  667. break;
  668. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  669. case VP9PROFILE_PROFILE0:
  670. case VP9PROFILE_PROFILE2:
  671. codec_ = VideoCodec::kVP9;
  672. break;
  673. default:
  674. NOTREACHED() << "Unsupported profile.";
  675. };
  676. // Count the session as successfully initialized.
  677. UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
  678. SFT_SUCCESSFULLY_INITIALIZED, SFT_MAX + 1);
  679. return true;
  680. }
  681. bool VTVideoDecodeAccelerator::FinishDelayedFrames() {
  682. DVLOG(3) << __func__;
  683. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  684. if (session_) {
  685. OSStatus status = VTDecompressionSessionWaitForAsynchronousFrames(session_);
  686. output_count_for_cra_rasl_workaround_ = 0;
  687. if (status) {
  688. NOTIFY_STATUS("VTDecompressionSessionWaitForAsynchronousFrames()", status,
  689. SFT_PLATFORM_ERROR);
  690. return false;
  691. }
  692. }
  693. return true;
  694. }
  695. bool VTVideoDecodeAccelerator::ConfigureDecoder() {
  696. DVLOG(2) << __func__;
  697. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  698. base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
  699. switch (codec_) {
  700. case VideoCodec::kH264:
  701. format = CreateVideoFormatH264(active_sps_, active_spsext_, active_pps_);
  702. break;
  703. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  704. case VideoCodec::kHEVC:
  705. format = CreateVideoFormatHEVC(active_vps_, active_sps_, active_pps_);
  706. break;
  707. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  708. case VideoCodec::kVP9:
  709. format = CreateVideoFormatVP9(
  710. cc_detector_->GetColorSpace(config_.container_color_space),
  711. config_.profile, config_.hdr_metadata,
  712. cc_detector_->GetCodedSize(config_.initial_expected_coded_size));
  713. break;
  714. default:
  715. NOTREACHED() << "Unsupported codec.";
  716. }
  717. if (!format) {
  718. NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
  719. return false;
  720. }
  721. // TODO(crbug.com/1103432): We should use
  722. // VTDecompressionSessionCanAcceptFormatDescription() on |format| here to
  723. // avoid the configuration change if possible.
  724. // Ensure that the old decoder emits all frames before the new decoder can
  725. // emit any.
  726. if (!FinishDelayedFrames())
  727. return false;
  728. format_ = format;
  729. session_.reset();
  730. // Note: We can always require hardware once Flash and PPAPI are gone.
  731. // however for HEVC, do not use require hardware to avoid
  732. // kVTVideoDecoderUnsupportedDataFormatErr(-12910) failure,
  733. // hardware acceleration may be unavailable for a number of reasons,
  734. // so just enable hardware and let vt choose whether to use
  735. // hardware of software decode
  736. const bool require_hardware = config_.profile == VP9PROFILE_PROFILE0 ||
  737. config_.profile == VP9PROFILE_PROFILE2;
  738. const bool is_hbd = config_.profile == VP9PROFILE_PROFILE2 ||
  739. config_.profile == HEVCPROFILE_MAIN10 ||
  740. config_.profile == HEVCPROFILE_REXT;
  741. if (!CreateVideoToolboxSession(format_, require_hardware, is_hbd, &callback_,
  742. &session_, &configured_size_)) {
  743. NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
  744. return false;
  745. }
  746. // Report whether hardware decode is being used.
  747. bool using_hardware = false;
  748. base::ScopedCFTypeRef<CFBooleanRef> cf_using_hardware;
  749. if (VTSessionCopyProperty(
  750. session_,
  751. // kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder
  752. CFSTR("UsingHardwareAcceleratedVideoDecoder"), kCFAllocatorDefault,
  753. cf_using_hardware.InitializeInto()) == 0) {
  754. using_hardware = CFBooleanGetValue(cf_using_hardware);
  755. }
  756. UMA_HISTOGRAM_BOOLEAN("Media.VTVDA.HardwareAccelerated", using_hardware);
  757. if (codec_ == VideoCodec::kVP9 && !vp9_bsf_)
  758. vp9_bsf_ = std::make_unique<VP9SuperFrameBitstreamFilter>();
  759. // Record that the configuration change is complete.
  760. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  761. configured_vps_ = active_vps_;
  762. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  763. configured_sps_ = active_sps_;
  764. configured_spsext_ = active_spsext_;
  765. configured_pps_ = active_pps_;
  766. return true;
  767. }
  768. void VTVideoDecodeAccelerator::DecodeTaskVp9(
  769. scoped_refptr<DecoderBuffer> buffer,
  770. Frame* frame) {
  771. DVLOG(2) << __func__ << ": bit_stream=" << frame->bitstream_id
  772. << ", buffer=" << buffer->AsHumanReadableString();
  773. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  774. if (!cc_detector_)
  775. cc_detector_ = std::make_unique<VP9ConfigChangeDetector>();
  776. cc_detector_->DetectConfig(buffer->data(), buffer->data_size());
  777. if (!session_ || cc_detector_->config_changed()) {
  778. // ConfigureDecoder() calls NotifyError() on failure.
  779. if (!ConfigureDecoder())
  780. return;
  781. }
  782. // Now that the configuration is up to date, copy it into the frame.
  783. frame->image_size = configured_size_;
  784. if (!vp9_bsf_->EnqueueBuffer(std::move(buffer))) {
  785. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported VP9 stream");
  786. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  787. return;
  788. }
  789. // If we have no buffer this bitstream buffer is part of a super frame that we
  790. // need to assemble before giving to VideoToolbox.
  791. auto data = vp9_bsf_->take_buffer();
  792. if (!data) {
  793. gpu_task_runner_->PostTask(
  794. FROM_HERE, base::BindOnce(&VTVideoDecodeAccelerator::DecodeDone,
  795. weak_this_, frame));
  796. return;
  797. }
  798. // Package the data in a CMSampleBuffer.
  799. base::ScopedCFTypeRef<CMSampleBufferRef> sample;
  800. OSStatus status = CMSampleBufferCreateReady(kCFAllocatorDefault,
  801. data, // data_buffer
  802. format_, // format_description
  803. 1, // num_samples
  804. 0, // num_sample_timing_entries
  805. nullptr, // &sample_timing_array
  806. 0, // num_sample_size_entries
  807. nullptr, // &sample_size_array
  808. sample.InitializeInto());
  809. if (status) {
  810. NOTIFY_STATUS("CMSampleBufferCreate()", status, SFT_PLATFORM_ERROR);
  811. return;
  812. }
  813. // Send the frame for decoding.
  814. // Asynchronous Decompression allows for parallel submission of frames
  815. // (without it, DecodeFrame() does not return until the frame has been
  816. // decoded). We don't enable Temporal Processing because we are not passing
  817. // timestamps anyway.
  818. VTDecodeFrameFlags decode_flags =
  819. kVTDecodeFrame_EnableAsynchronousDecompression;
  820. status = VTDecompressionSessionDecodeFrame(
  821. session_,
  822. sample, // sample_buffer
  823. decode_flags, // decode_flags
  824. reinterpret_cast<void*>(frame), // source_frame_refcon
  825. nullptr); // &info_flags_out
  826. if (status) {
  827. NOTIFY_STATUS("VTDecompressionSessionDecodeFrame()", status,
  828. SFT_DECODE_ERROR);
  829. return;
  830. }
  831. }
  832. void VTVideoDecodeAccelerator::DecodeTaskH264(
  833. scoped_refptr<DecoderBuffer> buffer,
  834. Frame* frame) {
  835. DVLOG(2) << __func__ << "(" << frame->bitstream_id << ")";
  836. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  837. // NALUs are stored with Annex B format in the bitstream buffer (start codes),
  838. // but VideoToolbox expects AVC format (length headers), so we must rewrite
  839. // the data.
  840. //
  841. // Locate relevant NALUs and compute the size of the rewritten data. Also
  842. // record parameter sets for VideoToolbox initialization.
  843. size_t data_size = 0;
  844. std::vector<H264NALU> nalus;
  845. size_t first_slice_index = 0;
  846. h264_parser_.SetStream(buffer->data(), buffer->data_size());
  847. H264NALU nalu;
  848. while (true) {
  849. H264Parser::Result result = h264_parser_.AdvanceToNextNALU(&nalu);
  850. if (result == H264Parser::kEOStream)
  851. break;
  852. if (result == H264Parser::kUnsupportedStream) {
  853. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported H.264 stream");
  854. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  855. return;
  856. }
  857. if (result != H264Parser::kOk) {
  858. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  859. "Failed to parse H.264 stream");
  860. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  861. return;
  862. }
  863. switch (nalu.nal_unit_type) {
  864. case H264NALU::kSPS: {
  865. int sps_id = -1;
  866. result = h264_parser_.ParseSPS(&sps_id);
  867. if (result == H264Parser::kUnsupportedStream) {
  868. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported SPS");
  869. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  870. return;
  871. }
  872. if (result != H264Parser::kOk) {
  873. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Could not parse SPS");
  874. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  875. return;
  876. }
  877. seen_sps_[sps_id].assign(nalu.data, nalu.data + nalu.size);
  878. seen_spsext_.erase(sps_id);
  879. break;
  880. }
  881. case H264NALU::kSPSExt: {
  882. int sps_id = -1;
  883. result = h264_parser_.ParseSPSExt(&sps_id);
  884. if (result != H264Parser::kOk) {
  885. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  886. "Could not parse SPS extension");
  887. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  888. return;
  889. }
  890. seen_spsext_[sps_id].assign(nalu.data, nalu.data + nalu.size);
  891. break;
  892. }
  893. case H264NALU::kPPS: {
  894. int pps_id = -1;
  895. result = h264_parser_.ParsePPS(&pps_id);
  896. if (result == H264Parser::kUnsupportedStream) {
  897. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported PPS");
  898. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  899. return;
  900. }
  901. if (result != H264Parser::kOk) {
  902. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Could not parse PPS");
  903. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  904. return;
  905. }
  906. seen_pps_[pps_id].assign(nalu.data, nalu.data + nalu.size);
  907. break;
  908. }
  909. case H264NALU::kSEIMessage: {
  910. H264SEIMessage sei_msg;
  911. result = h264_parser_.ParseSEI(&sei_msg);
  912. if (result == H264Parser::kOk &&
  913. sei_msg.type == H264SEIMessage::kSEIRecoveryPoint &&
  914. sei_msg.recovery_point.recovery_frame_cnt == 0) {
  915. // We only support immediate recovery points. Supporting future points
  916. // would require dropping |recovery_frame_cnt| frames when needed.
  917. frame->has_recovery_point = true;
  918. }
  919. nalus.push_back(nalu);
  920. data_size += kNALUHeaderLength + nalu.size;
  921. break;
  922. }
  923. case H264NALU::kSliceDataA:
  924. case H264NALU::kSliceDataB:
  925. case H264NALU::kSliceDataC:
  926. case H264NALU::kNonIDRSlice:
  927. case H264NALU::kIDRSlice:
  928. // Only the first slice is examined. Other slices are at least one of:
  929. // the same frame, not decoded, invalid.
  930. if (!frame->has_slice) {
  931. // Parse slice header.
  932. H264SliceHeader slice_hdr;
  933. result = h264_parser_.ParseSliceHeader(nalu, &slice_hdr);
  934. if (result == H264Parser::kUnsupportedStream) {
  935. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  936. "Unsupported slice header");
  937. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  938. return;
  939. }
  940. if (result != H264Parser::kOk) {
  941. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  942. "Could not parse slice header");
  943. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  944. return;
  945. }
  946. // Lookup SPS and PPS.
  947. const H264PPS* pps =
  948. h264_parser_.GetPPS(slice_hdr.pic_parameter_set_id);
  949. if (!pps) {
  950. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  951. "Missing PPS referenced by slice");
  952. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  953. return;
  954. }
  955. const H264SPS* sps = h264_parser_.GetSPS(pps->seq_parameter_set_id);
  956. if (!sps) {
  957. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  958. "Missing SPS referenced by PPS");
  959. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  960. return;
  961. }
  962. // Record the configuration.
  963. DCHECK(seen_pps_.contains(slice_hdr.pic_parameter_set_id));
  964. DCHECK(seen_sps_.contains(pps->seq_parameter_set_id));
  965. active_sps_ = seen_sps_[pps->seq_parameter_set_id];
  966. // Note: SPS extension lookup may create an empty entry.
  967. active_spsext_ = seen_spsext_[pps->seq_parameter_set_id];
  968. active_pps_ = seen_pps_[slice_hdr.pic_parameter_set_id];
  969. // Compute and store frame properties. |image_size| gets filled in
  970. // later, since it comes from the decoder configuration.
  971. absl::optional<int32_t> pic_order_cnt =
  972. h264_poc_.ComputePicOrderCnt(sps, slice_hdr);
  973. if (!pic_order_cnt.has_value()) {
  974. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  975. "Unable to compute POC");
  976. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  977. return;
  978. }
  979. frame->has_slice = true;
  980. frame->is_idr = nalu.nal_unit_type == media::H264NALU::kIDRSlice;
  981. frame->has_mmco5 = h264_poc_.IsPendingMMCO5();
  982. frame->pic_order_cnt = *pic_order_cnt;
  983. frame->reorder_window = ComputeH264ReorderWindow(sps);
  984. first_slice_index = nalus.size();
  985. }
  986. nalus.push_back(nalu);
  987. data_size += kNALUHeaderLength + nalu.size;
  988. break;
  989. default:
  990. nalus.push_back(nalu);
  991. data_size += kNALUHeaderLength + nalu.size;
  992. break;
  993. }
  994. }
  995. if (frame->is_idr || frame->has_recovery_point)
  996. waiting_for_idr_ = false;
  997. // If no IDR has been seen yet, skip decoding. Note that Flash sends
  998. // configuration changes as a bitstream with only SPS/PPS; we don't print
  999. // error messages for those.
  1000. if (frame->has_slice && waiting_for_idr_) {
  1001. if (!missing_idr_logged_) {
  1002. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1003. ("Illegal attempt to decode without IDR. "
  1004. "Discarding decode requests until the next IDR."));
  1005. missing_idr_logged_ = true;
  1006. }
  1007. frame->has_slice = false;
  1008. }
  1009. // If there is nothing to decode, drop the request by returning a frame with
  1010. // no image.
  1011. if (!frame->has_slice) {
  1012. gpu_task_runner_->PostTask(
  1013. FROM_HERE, base::BindOnce(&VTVideoDecodeAccelerator::DecodeDone,
  1014. weak_this_, frame));
  1015. return;
  1016. }
  1017. // If the configuration has changed, and we're at an IDR, reconfigure the
  1018. // decoding session. Otherwise insert the parameter sets and hope for the
  1019. // best.
  1020. if (configured_sps_ != active_sps_ || configured_spsext_ != active_spsext_ ||
  1021. configured_pps_ != active_pps_) {
  1022. if (frame->is_idr) {
  1023. if (active_sps_.empty()) {
  1024. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1025. "Invalid configuration (no SPS)");
  1026. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1027. return;
  1028. }
  1029. if (active_pps_.empty()) {
  1030. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1031. "Invalid configuration (no PPS)");
  1032. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1033. return;
  1034. }
  1035. // ConfigureDecoder() calls NotifyError() on failure.
  1036. if (!ConfigureDecoder())
  1037. return;
  1038. } else {
  1039. // Only |data| and |size| are read later, other fields are left empty.
  1040. media::H264NALU sps_nalu;
  1041. sps_nalu.data = active_sps_.data();
  1042. sps_nalu.size = active_sps_.size();
  1043. nalus.insert(nalus.begin() + first_slice_index, sps_nalu);
  1044. data_size += kNALUHeaderLength + sps_nalu.size;
  1045. first_slice_index += 1;
  1046. if (active_spsext_.size()) {
  1047. media::H264NALU spsext_nalu;
  1048. spsext_nalu.data = active_spsext_.data();
  1049. spsext_nalu.size = active_spsext_.size();
  1050. nalus.insert(nalus.begin() + first_slice_index, spsext_nalu);
  1051. data_size += kNALUHeaderLength + spsext_nalu.size;
  1052. first_slice_index += 1;
  1053. }
  1054. media::H264NALU pps_nalu;
  1055. pps_nalu.data = active_pps_.data();
  1056. pps_nalu.size = active_pps_.size();
  1057. nalus.insert(nalus.begin() + first_slice_index, pps_nalu);
  1058. data_size += kNALUHeaderLength + pps_nalu.size;
  1059. first_slice_index += 1;
  1060. }
  1061. }
  1062. // If the session is not configured by this point, fail.
  1063. if (!session_) {
  1064. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1065. "Cannot decode without configuration");
  1066. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1067. return;
  1068. }
  1069. // Now that the configuration is up to date, copy it into the frame.
  1070. frame->image_size = configured_size_;
  1071. // Create a memory-backed CMBlockBuffer for the translated data.
  1072. // TODO(sandersd): Pool of memory blocks.
  1073. base::ScopedCFTypeRef<CMBlockBufferRef> data;
  1074. OSStatus status = CMBlockBufferCreateWithMemoryBlock(
  1075. kCFAllocatorDefault,
  1076. nullptr, // &memory_block
  1077. data_size, // block_length
  1078. kCFAllocatorDefault, // block_allocator
  1079. nullptr, // &custom_block_source
  1080. 0, // offset_to_data
  1081. data_size, // data_length
  1082. 0, // flags
  1083. data.InitializeInto());
  1084. if (status) {
  1085. NOTIFY_STATUS("CMBlockBufferCreateWithMemoryBlock()", status,
  1086. SFT_PLATFORM_ERROR);
  1087. return;
  1088. }
  1089. // Make sure that the memory is actually allocated.
  1090. // CMBlockBufferReplaceDataBytes() is documented to do this, but prints a
  1091. // message each time starting in Mac OS X 10.10.
  1092. status = CMBlockBufferAssureBlockMemory(data);
  1093. if (status) {
  1094. NOTIFY_STATUS("CMBlockBufferAssureBlockMemory()", status,
  1095. SFT_PLATFORM_ERROR);
  1096. return;
  1097. }
  1098. // Copy NALU data into the CMBlockBuffer, inserting length headers.
  1099. size_t offset = 0;
  1100. for (size_t i = 0; i < nalus.size(); i++) {
  1101. H264NALU& nalu_ref = nalus[i];
  1102. uint32_t header = base::HostToNet32(static_cast<uint32_t>(nalu_ref.size));
  1103. status =
  1104. CMBlockBufferReplaceDataBytes(&header, data, offset, kNALUHeaderLength);
  1105. if (status) {
  1106. NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
  1107. SFT_PLATFORM_ERROR);
  1108. return;
  1109. }
  1110. offset += kNALUHeaderLength;
  1111. status = CMBlockBufferReplaceDataBytes(nalu_ref.data, data, offset,
  1112. nalu_ref.size);
  1113. if (status) {
  1114. NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
  1115. SFT_PLATFORM_ERROR);
  1116. return;
  1117. }
  1118. offset += nalu_ref.size;
  1119. }
  1120. // Package the data in a CMSampleBuffer.
  1121. base::ScopedCFTypeRef<CMSampleBufferRef> sample;
  1122. status = CMSampleBufferCreate(kCFAllocatorDefault,
  1123. data, // data_buffer
  1124. true, // data_ready
  1125. nullptr, // make_data_ready_callback
  1126. nullptr, // make_data_ready_refcon
  1127. format_, // format_description
  1128. 1, // num_samples
  1129. 0, // num_sample_timing_entries
  1130. nullptr, // &sample_timing_array
  1131. 1, // num_sample_size_entries
  1132. &data_size, // &sample_size_array
  1133. sample.InitializeInto());
  1134. if (status) {
  1135. NOTIFY_STATUS("CMSampleBufferCreate()", status, SFT_PLATFORM_ERROR);
  1136. return;
  1137. }
  1138. // Send the frame for decoding.
  1139. // Asynchronous Decompression allows for parallel submission of frames
  1140. // (without it, DecodeFrame() does not return until the frame has been
  1141. // decoded). We don't enable Temporal Processing because we are not passing
  1142. // timestamps anyway.
  1143. VTDecodeFrameFlags decode_flags =
  1144. kVTDecodeFrame_EnableAsynchronousDecompression;
  1145. status = VTDecompressionSessionDecodeFrame(
  1146. session_,
  1147. sample, // sample_buffer
  1148. decode_flags, // decode_flags
  1149. reinterpret_cast<void*>(frame), // source_frame_refcon
  1150. nullptr); // &info_flags_out
  1151. if (status) {
  1152. NOTIFY_STATUS("VTDecompressionSessionDecodeFrame()", status,
  1153. SFT_DECODE_ERROR);
  1154. return;
  1155. }
  1156. }
  1157. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  1158. void VTVideoDecodeAccelerator::DecodeTaskHEVC(
  1159. scoped_refptr<DecoderBuffer> buffer,
  1160. Frame* frame) {
  1161. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  1162. size_t data_size = 0;
  1163. std::vector<H265NALU> nalus;
  1164. // store the previous slice header to allow us recover the current
  1165. // from the previous one if dependent_slice_segment_flag exists
  1166. std::unique_ptr<H265SliceHeader> curr_slice_hdr;
  1167. std::unique_ptr<H265SliceHeader> last_slice_hdr;
  1168. hevc_parser_.SetStream(buffer->data(), buffer->data_size());
  1169. H265NALU nalu;
  1170. while (true) {
  1171. H265Parser::Result result = hevc_parser_.AdvanceToNextNALU(&nalu);
  1172. if (result == H265Parser::kEOStream) {
  1173. last_slice_hdr.reset();
  1174. break;
  1175. }
  1176. if (result == H265Parser::kUnsupportedStream) {
  1177. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported H.265 stream");
  1178. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  1179. return;
  1180. }
  1181. if (result != H265Parser::kOk) {
  1182. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1183. "Failed to parse H.265 stream");
  1184. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1185. return;
  1186. }
  1187. // 8.1.2 We only want nuh_layer_id of zero.
  1188. if (nalu.nuh_layer_id) {
  1189. MEDIA_LOG(INFO, media_log_)
  1190. << "Skipping NALU with nuh_layer_id=" << nalu.nuh_layer_id;
  1191. continue;
  1192. }
  1193. switch (nalu.nal_unit_type) {
  1194. case H265NALU::SPS_NUT: {
  1195. int sps_id = -1;
  1196. result = hevc_parser_.ParseSPS(&sps_id);
  1197. if (result == H265Parser::kUnsupportedStream) {
  1198. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported SPS");
  1199. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  1200. return;
  1201. }
  1202. if (result != H265Parser::kOk) {
  1203. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Could not parse SPS");
  1204. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1205. return;
  1206. }
  1207. seen_sps_[sps_id].assign(nalu.data, nalu.data + nalu.size);
  1208. break;
  1209. }
  1210. case H265NALU::PPS_NUT: {
  1211. int pps_id = -1;
  1212. result = hevc_parser_.ParsePPS(nalu, &pps_id);
  1213. if (result == H265Parser::kUnsupportedStream) {
  1214. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported PPS");
  1215. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  1216. return;
  1217. }
  1218. if (result == H265Parser::kMissingParameterSet) {
  1219. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1220. "Missing SPS from what was parsed");
  1221. NotifyError(PLATFORM_FAILURE, SFT_INVALID_STREAM);
  1222. return;
  1223. }
  1224. if (result != H265Parser::kOk) {
  1225. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Could not parse PPS");
  1226. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1227. return;
  1228. }
  1229. seen_pps_[pps_id].assign(nalu.data, nalu.data + nalu.size);
  1230. // Pass PPS as data to the platform decoder, it helps in cases
  1231. // when there are more than one PPS, Video Toolbox is smart enough
  1232. // to find and recognize them there.
  1233. nalus.push_back(nalu);
  1234. data_size += kNALUHeaderLength + nalu.size;
  1235. break;
  1236. }
  1237. case H265NALU::VPS_NUT: {
  1238. int vps_id = -1;
  1239. result = hevc_parser_.ParseVPS(&vps_id);
  1240. if (result == H265Parser::kUnsupportedStream) {
  1241. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Unsupported VPS");
  1242. NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
  1243. return;
  1244. }
  1245. if (result != H265Parser::kOk) {
  1246. WriteToMediaLog(MediaLogMessageLevel::kERROR, "Could not parse VPS");
  1247. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1248. return;
  1249. }
  1250. seen_vps_[vps_id].assign(nalu.data, nalu.data + nalu.size);
  1251. break;
  1252. }
  1253. case H265NALU::EOS_NUT:
  1254. hevc_poc_.Reset();
  1255. nalus.push_back(nalu);
  1256. data_size += kNALUHeaderLength + nalu.size;
  1257. break;
  1258. case H265NALU::BLA_W_LP:
  1259. case H265NALU::BLA_W_RADL:
  1260. case H265NALU::BLA_N_LP:
  1261. case H265NALU::IDR_W_RADL:
  1262. case H265NALU::IDR_N_LP:
  1263. case H265NALU::TRAIL_N:
  1264. case H265NALU::TRAIL_R:
  1265. case H265NALU::TSA_N:
  1266. case H265NALU::TSA_R:
  1267. case H265NALU::STSA_N:
  1268. case H265NALU::STSA_R:
  1269. case H265NALU::RADL_N:
  1270. case H265NALU::RADL_R:
  1271. case H265NALU::RASL_N:
  1272. case H265NALU::RASL_R:
  1273. case H265NALU::CRA_NUT: {
  1274. // The VT session will report a OsStatus=12909 kVTVideoDecoderBadDataErr
  1275. // if you send a RASL frame just after a CRA frame, so we wait until the
  1276. // total output count is enough
  1277. if (output_count_for_cra_rasl_workaround_ < kMinOutputsBeforeRASL &&
  1278. (nalu.nal_unit_type == H265NALU::RASL_N ||
  1279. nalu.nal_unit_type == H265NALU::RASL_R)) {
  1280. continue;
  1281. }
  1282. // Just like H264, only the first slice is examined. Other slices are at
  1283. // least one of: the same frame, not decoded, invalid so no need to
  1284. // parse again.
  1285. if (frame->has_slice) {
  1286. nalus.push_back(nalu);
  1287. data_size += kNALUHeaderLength + nalu.size;
  1288. break;
  1289. }
  1290. curr_slice_hdr.reset(new H265SliceHeader());
  1291. result = hevc_parser_.ParseSliceHeader(nalu, curr_slice_hdr.get(),
  1292. last_slice_hdr.get());
  1293. if (result == H265Parser::kMissingParameterSet) {
  1294. curr_slice_hdr.reset();
  1295. last_slice_hdr.reset();
  1296. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1297. "Missing PPS when parsing slice header");
  1298. continue;
  1299. }
  1300. if (result != H265Parser::kOk) {
  1301. curr_slice_hdr.reset();
  1302. last_slice_hdr.reset();
  1303. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1304. "Could not parse slice header");
  1305. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1306. return;
  1307. }
  1308. const H265PPS* pps =
  1309. hevc_parser_.GetPPS(curr_slice_hdr->slice_pic_parameter_set_id);
  1310. if (!pps) {
  1311. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1312. "Missing PPS referenced by slice");
  1313. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1314. return;
  1315. }
  1316. const H265SPS* sps = hevc_parser_.GetSPS(pps->pps_seq_parameter_set_id);
  1317. if (!sps) {
  1318. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1319. "Missing SPS referenced by PPS");
  1320. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1321. return;
  1322. }
  1323. const H265VPS* vps =
  1324. hevc_parser_.GetVPS(sps->sps_video_parameter_set_id);
  1325. if (!vps) {
  1326. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1327. "Missing VPS referenced by SPS");
  1328. NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
  1329. return;
  1330. }
  1331. // Record the configuration.
  1332. DCHECK(seen_pps_.contains(curr_slice_hdr->slice_pic_parameter_set_id));
  1333. DCHECK(seen_sps_.contains(pps->pps_seq_parameter_set_id));
  1334. DCHECK(seen_vps_.contains(sps->sps_video_parameter_set_id));
  1335. active_vps_ = seen_vps_[sps->sps_video_parameter_set_id];
  1336. active_sps_ = seen_sps_[pps->pps_seq_parameter_set_id];
  1337. active_pps_ = seen_pps_[curr_slice_hdr->slice_pic_parameter_set_id];
  1338. // Compute and store frame properties. |image_size| gets filled in
  1339. // later, since it comes from the decoder configuration.
  1340. int32_t pic_order_cnt =
  1341. hevc_poc_.ComputePicOrderCnt(sps, pps, *curr_slice_hdr.get());
  1342. frame->has_slice = true;
  1343. frame->is_idr = nalu.nal_unit_type >= H265NALU::BLA_W_LP &&
  1344. nalu.nal_unit_type <= H265NALU::RSV_IRAP_VCL23;
  1345. frame->pic_order_cnt = pic_order_cnt;
  1346. frame->reorder_window = ComputeHEVCReorderWindow(vps);
  1347. last_slice_hdr.swap(curr_slice_hdr);
  1348. curr_slice_hdr.reset();
  1349. [[fallthrough]];
  1350. }
  1351. default:
  1352. nalus.push_back(nalu);
  1353. data_size += kNALUHeaderLength + nalu.size;
  1354. break;
  1355. }
  1356. }
  1357. if (frame->is_idr)
  1358. waiting_for_idr_ = false;
  1359. // If no IDR has been seen yet, skip decoding. Note that Flash sends
  1360. // configuration changes as a bitstream with only SPS/PPS; we don't print
  1361. // error messages for those.
  1362. if (frame->has_slice && waiting_for_idr_) {
  1363. if (!missing_idr_logged_) {
  1364. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1365. ("Illegal attempt to decode without IDR. "
  1366. "Discarding decode requests until the next IDR."));
  1367. missing_idr_logged_ = true;
  1368. }
  1369. frame->has_slice = false;
  1370. }
  1371. // If there is nothing to decode, drop the request by returning a frame with
  1372. // no image.
  1373. if (!frame->has_slice) {
  1374. gpu_task_runner_->PostTask(
  1375. FROM_HERE, base::BindOnce(&VTVideoDecodeAccelerator::DecodeDone,
  1376. weak_this_, frame));
  1377. return;
  1378. }
  1379. // Apply any configuration change, but only at an IDR. If there is no IDR, we
  1380. // just hope for the best from the decoder.
  1381. if (frame->is_idr &&
  1382. (configured_vps_ != active_vps_ || configured_sps_ != active_sps_ ||
  1383. configured_pps_ != active_pps_)) {
  1384. if (active_vps_.empty()) {
  1385. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1386. "Invalid configuration (no VPS)");
  1387. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1388. return;
  1389. }
  1390. if (active_sps_.empty()) {
  1391. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1392. "Invalid configuration (no SPS)");
  1393. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1394. return;
  1395. }
  1396. if (active_pps_.empty()) {
  1397. WriteToMediaLog(MediaLogMessageLevel::kERROR,
  1398. "Invalid configuration (no PPS)");
  1399. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1400. return;
  1401. }
  1402. // ConfigureDecoder() calls NotifyError() on failure.
  1403. if (!ConfigureDecoder()) {
  1404. return;
  1405. }
  1406. }
  1407. // Now that the configuration is up to date, copy it into the frame.
  1408. frame->image_size = configured_size_;
  1409. // Create a memory-backed CMBlockBuffer for the translated data.
  1410. // TODO(sandersd): Pool of memory blocks.
  1411. base::ScopedCFTypeRef<CMBlockBufferRef> data;
  1412. OSStatus status = CMBlockBufferCreateWithMemoryBlock(
  1413. kCFAllocatorDefault,
  1414. nullptr, // &memory_block
  1415. data_size, // block_length
  1416. kCFAllocatorDefault, // block_allocator
  1417. nullptr, // &custom_block_source
  1418. 0, // offset_to_data
  1419. data_size, // data_length
  1420. 0, // flags
  1421. data.InitializeInto());
  1422. if (status) {
  1423. NOTIFY_STATUS("CMBlockBufferCreateWithMemoryBlock()", status,
  1424. SFT_PLATFORM_ERROR);
  1425. return;
  1426. }
  1427. // Make sure that the memory is actually allocated.
  1428. // CMBlockBufferReplaceDataBytes() is documented to do this, but prints a
  1429. // message each time starting in Mac OS X 10.10.
  1430. status = CMBlockBufferAssureBlockMemory(data);
  1431. if (status) {
  1432. NOTIFY_STATUS("CMBlockBufferAssureBlockMemory()", status,
  1433. SFT_PLATFORM_ERROR);
  1434. return;
  1435. }
  1436. // Copy NALU data into the CMBlockBuffer, inserting length headers.
  1437. size_t offset = 0;
  1438. for (size_t i = 0; i < nalus.size(); i++) {
  1439. H265NALU& nalu_ref = nalus[i];
  1440. uint32_t header = base::HostToNet32(static_cast<uint32_t>(nalu_ref.size));
  1441. status =
  1442. CMBlockBufferReplaceDataBytes(&header, data, offset, kNALUHeaderLength);
  1443. if (status) {
  1444. NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
  1445. SFT_PLATFORM_ERROR);
  1446. return;
  1447. }
  1448. offset += kNALUHeaderLength;
  1449. status = CMBlockBufferReplaceDataBytes(nalu_ref.data, data, offset,
  1450. nalu_ref.size);
  1451. if (status) {
  1452. NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
  1453. SFT_PLATFORM_ERROR);
  1454. return;
  1455. }
  1456. offset += nalu_ref.size;
  1457. }
  1458. // Package the data in a CMSampleBuffer.
  1459. base::ScopedCFTypeRef<CMSampleBufferRef> sample;
  1460. status = CMSampleBufferCreate(kCFAllocatorDefault,
  1461. data, // data_buffer
  1462. true, // data_ready
  1463. nullptr, // make_data_ready_callback
  1464. nullptr, // make_data_ready_refcon
  1465. format_, // format_description
  1466. 1, // num_samples
  1467. 0, // num_sample_timing_entries
  1468. nullptr, // &sample_timing_array
  1469. 1, // num_sample_size_entries
  1470. &data_size, // &sample_size_array
  1471. sample.InitializeInto());
  1472. if (status) {
  1473. NOTIFY_STATUS("CMSampleBufferCreate()", status, SFT_PLATFORM_ERROR);
  1474. return;
  1475. }
  1476. // Send the frame for decoding.
  1477. // Asynchronous Decompression allows for parallel submission of frames
  1478. // (without it, DecodeFrame() does not return until the frame has been
  1479. // decoded). We don't enable Temporal Processing because we are not passing
  1480. // timestamps anyway.
  1481. VTDecodeFrameFlags decode_flags =
  1482. kVTDecodeFrame_EnableAsynchronousDecompression;
  1483. status = VTDecompressionSessionDecodeFrame(
  1484. session_,
  1485. sample, // sample_buffer
  1486. decode_flags, // decode_flags
  1487. reinterpret_cast<void*>(frame), // source_frame_refcon
  1488. nullptr); // &info_flags_out
  1489. if (status) {
  1490. NOTIFY_STATUS("VTDecompressionSessionDecodeFrame()", status,
  1491. SFT_DECODE_ERROR);
  1492. return;
  1493. }
  1494. }
  1495. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  1496. // This method may be called on any VideoToolbox thread.
  1497. void VTVideoDecodeAccelerator::Output(void* source_frame_refcon,
  1498. OSStatus status,
  1499. CVImageBufferRef image_buffer) {
  1500. if (status) {
  1501. NOTIFY_STATUS("Decoding", status, SFT_DECODE_ERROR);
  1502. return;
  1503. }
  1504. // The type of |image_buffer| is CVImageBuffer, but we only handle
  1505. // CVPixelBuffers. This should be guaranteed as we set
  1506. // kCVPixelBufferOpenGLCompatibilityKey in |image_config|.
  1507. //
  1508. // Sometimes, for unknown reasons (http://crbug.com/453050), |image_buffer| is
  1509. // NULL, which causes CFGetTypeID() to crash. While the rest of the code would
  1510. // smoothly handle NULL as a dropped frame, we choose to fail permanantly here
  1511. // until the issue is better understood.
  1512. if (!image_buffer || CFGetTypeID(image_buffer) != CVPixelBufferGetTypeID()) {
  1513. DLOG(ERROR) << "Decoded frame is not a CVPixelBuffer";
  1514. NotifyError(PLATFORM_FAILURE, SFT_DECODE_ERROR);
  1515. return;
  1516. }
  1517. Frame* frame = reinterpret_cast<Frame*>(source_frame_refcon);
  1518. frame->image.reset(image_buffer, base::scoped_policy::RETAIN);
  1519. gpu_task_runner_->PostTask(
  1520. FROM_HERE,
  1521. base::BindOnce(&VTVideoDecodeAccelerator::DecodeDone, weak_this_, frame));
  1522. }
  1523. void VTVideoDecodeAccelerator::DecodeDone(Frame* frame) {
  1524. DVLOG(3) << __func__ << "(" << frame->bitstream_id << ")";
  1525. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1526. // pending_frames_.erase() will delete |frame|.
  1527. int32_t bitstream_id = frame->bitstream_id;
  1528. DCHECK(pending_frames_.contains(bitstream_id));
  1529. if (state_ == STATE_ERROR || state_ == STATE_DESTROYING) {
  1530. // Destroy() handles NotifyEndOfBitstreamBuffer().
  1531. pending_frames_.erase(bitstream_id);
  1532. return;
  1533. }
  1534. DCHECK_EQ(state_, STATE_DECODING);
  1535. if (!frame->image.get()) {
  1536. pending_frames_.erase(bitstream_id);
  1537. assigned_bitstream_ids_.erase(bitstream_id);
  1538. client_->NotifyEndOfBitstreamBuffer(bitstream_id);
  1539. return;
  1540. }
  1541. output_count_for_cra_rasl_workaround_++;
  1542. Task task(TASK_FRAME);
  1543. task.frame = std::move(pending_frames_[bitstream_id]);
  1544. pending_frames_.erase(bitstream_id);
  1545. task_queue_.push(std::move(task));
  1546. ProcessWorkQueues();
  1547. }
  1548. void VTVideoDecodeAccelerator::FlushTask(TaskType type) {
  1549. DVLOG(3) << __func__;
  1550. DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
  1551. FinishDelayedFrames();
  1552. // All the frames that are going to be sent must have been sent by now. So
  1553. // clear any state in the bitstream filter.
  1554. if (vp9_bsf_)
  1555. vp9_bsf_->Flush();
  1556. if (type == TASK_DESTROY) {
  1557. if (session_) {
  1558. // Destroy the decoding session before returning from the decoder thread.
  1559. VTDecompressionSessionInvalidate(session_);
  1560. session_.reset();
  1561. }
  1562. // This must be done on |decoder_task_runner_|.
  1563. decoder_weak_this_factory_.InvalidateWeakPtrs();
  1564. }
  1565. // Queue a task even if flushing fails, so that destruction always completes.
  1566. gpu_task_runner_->PostTask(
  1567. FROM_HERE,
  1568. base::BindOnce(&VTVideoDecodeAccelerator::FlushDone, weak_this_, type));
  1569. }
  1570. void VTVideoDecodeAccelerator::FlushDone(TaskType type) {
  1571. DVLOG(3) << __func__;
  1572. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1573. task_queue_.push(Task(type));
  1574. ProcessWorkQueues();
  1575. }
  1576. void VTVideoDecodeAccelerator::Decode(BitstreamBuffer bitstream) {
  1577. Decode(bitstream.ToDecoderBuffer(), bitstream.id());
  1578. }
  1579. void VTVideoDecodeAccelerator::Decode(scoped_refptr<DecoderBuffer> buffer,
  1580. int32_t bitstream_id) {
  1581. DVLOG(2) << __func__ << "(" << bitstream_id << ")";
  1582. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1583. if (bitstream_id < 0) {
  1584. DLOG(ERROR) << "Invalid bitstream, id: " << bitstream_id;
  1585. NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
  1586. return;
  1587. }
  1588. if (!buffer) {
  1589. client_->NotifyEndOfBitstreamBuffer(bitstream_id);
  1590. return;
  1591. }
  1592. DCHECK_EQ(0u, assigned_bitstream_ids_.count(bitstream_id));
  1593. assigned_bitstream_ids_.insert(bitstream_id);
  1594. Frame* frame = new Frame(bitstream_id);
  1595. pending_frames_[bitstream_id] = base::WrapUnique(frame);
  1596. if (codec_ == VideoCodec::kVP9) {
  1597. decoder_task_runner_->PostTask(
  1598. FROM_HERE,
  1599. base::BindOnce(&VTVideoDecodeAccelerator::DecodeTaskVp9,
  1600. decoder_weak_this_, std::move(buffer), frame));
  1601. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  1602. } else if (codec_ == VideoCodec::kHEVC) {
  1603. decoder_task_runner_->PostTask(
  1604. FROM_HERE,
  1605. base::BindOnce(&VTVideoDecodeAccelerator::DecodeTaskHEVC,
  1606. decoder_weak_this_, std::move(buffer), frame));
  1607. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  1608. } else {
  1609. decoder_task_runner_->PostTask(
  1610. FROM_HERE,
  1611. base::BindOnce(&VTVideoDecodeAccelerator::DecodeTaskH264,
  1612. decoder_weak_this_, std::move(buffer), frame));
  1613. }
  1614. }
  1615. void VTVideoDecodeAccelerator::AssignPictureBuffers(
  1616. const std::vector<PictureBuffer>& pictures) {
  1617. DVLOG(1) << __func__;
  1618. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1619. for (const PictureBuffer& picture : pictures) {
  1620. DVLOG(3) << "AssignPictureBuffer(" << picture.id() << ")";
  1621. DCHECK(!picture_info_map_.contains(picture.id()));
  1622. assigned_picture_ids_.insert(picture.id());
  1623. available_picture_ids_.push_back(picture.id());
  1624. if (picture.client_texture_ids().empty() &&
  1625. picture.service_texture_ids().empty()) {
  1626. picture_info_map_.insert(
  1627. std::make_pair(picture.id(), std::make_unique<PictureInfo>()));
  1628. } else {
  1629. DCHECK_LE(1u, picture.client_texture_ids().size());
  1630. DCHECK_LE(1u, picture.service_texture_ids().size());
  1631. picture_info_map_.insert(std::make_pair(
  1632. picture.id(),
  1633. std::make_unique<PictureInfo>(picture.client_texture_ids()[0],
  1634. picture.service_texture_ids()[0])));
  1635. }
  1636. }
  1637. // Pictures are not marked as uncleared until after this method returns, and
  1638. // they will be broken if they are used before that happens. So, schedule
  1639. // future work after that happens.
  1640. gpu_task_runner_->PostTask(
  1641. FROM_HERE,
  1642. base::BindOnce(&VTVideoDecodeAccelerator::ProcessWorkQueues, weak_this_));
  1643. }
  1644. void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
  1645. DVLOG(2) << __func__ << "(" << picture_id << ")";
  1646. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1647. // It's possible there was a ReusePictureBuffer() request in flight when we
  1648. // called DismissPictureBuffer(), in which case we won't find it. In that case
  1649. // we should just drop the ReusePictureBuffer() request.
  1650. auto it = picture_info_map_.find(picture_id);
  1651. if (it == picture_info_map_.end())
  1652. return;
  1653. // Drop references to allow the underlying buffer to be released.
  1654. PictureInfo* picture_info = it->second.get();
  1655. if (picture_info->uses_shared_images) {
  1656. picture_info->scoped_shared_images.clear();
  1657. } else {
  1658. gl_client_.bind_image.Run(picture_info->client_texture_id,
  1659. gpu::GetPlatformSpecificTextureTarget(), nullptr,
  1660. false);
  1661. }
  1662. picture_info->gl_images.clear();
  1663. picture_info->bitstream_id = 0;
  1664. // Mark the picture as available and try to complete pending output work.
  1665. DCHECK(assigned_picture_ids_.count(picture_id));
  1666. available_picture_ids_.push_back(picture_id);
  1667. ProcessWorkQueues();
  1668. }
  1669. void VTVideoDecodeAccelerator::ProcessWorkQueues() {
  1670. DVLOG(3) << __func__;
  1671. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1672. switch (state_) {
  1673. case STATE_DECODING:
  1674. if (codec_ == VideoCodec::kVP9) {
  1675. while (state_ == STATE_DECODING) {
  1676. if (!ProcessOutputQueue() && !ProcessTaskQueue())
  1677. break;
  1678. }
  1679. return;
  1680. }
  1681. // TODO(sandersd): Batch where possible.
  1682. while (state_ == STATE_DECODING) {
  1683. if (!ProcessReorderQueue() && !ProcessTaskQueue())
  1684. break;
  1685. }
  1686. return;
  1687. case STATE_ERROR:
  1688. // Do nothing until Destroy() is called.
  1689. return;
  1690. case STATE_DESTROYING:
  1691. // Drop tasks until we are ready to destruct.
  1692. while (!task_queue_.empty()) {
  1693. if (task_queue_.front().type == TASK_DESTROY) {
  1694. delete this;
  1695. return;
  1696. }
  1697. task_queue_.pop();
  1698. }
  1699. return;
  1700. }
  1701. }
  1702. bool VTVideoDecodeAccelerator::ProcessTaskQueue() {
  1703. DVLOG(3) << __func__ << " size=" << task_queue_.size();
  1704. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1705. DCHECK_EQ(state_, STATE_DECODING);
  1706. if (task_queue_.empty())
  1707. return false;
  1708. Task& task = task_queue_.front();
  1709. switch (task.type) {
  1710. case TASK_FRAME: {
  1711. if (codec_ == VideoCodec::kVP9) {
  1712. // Once we've reached our maximum output queue size, defer end of
  1713. // bitstream buffer signals to avoid piling up too many frames.
  1714. if (output_queue_.size() >= limits::kMaxVideoFrames)
  1715. return false;
  1716. assigned_bitstream_ids_.erase(task.frame->bitstream_id);
  1717. client_->NotifyEndOfBitstreamBuffer(task.frame->bitstream_id);
  1718. output_queue_.push_back(std::move(task.frame));
  1719. task_queue_.pop();
  1720. return true;
  1721. }
  1722. bool reorder_queue_has_space =
  1723. reorder_queue_.size() < kMaxReorderQueueSize;
  1724. bool reorder_queue_flush_needed =
  1725. task.frame->is_idr || task.frame->has_mmco5;
  1726. bool reorder_queue_flush_done = reorder_queue_.empty();
  1727. if (reorder_queue_has_space &&
  1728. (!reorder_queue_flush_needed || reorder_queue_flush_done)) {
  1729. DVLOG(2) << "Decode(" << task.frame->bitstream_id << ") complete";
  1730. assigned_bitstream_ids_.erase(task.frame->bitstream_id);
  1731. client_->NotifyEndOfBitstreamBuffer(task.frame->bitstream_id);
  1732. reorder_queue_.push(std::move(task.frame));
  1733. task_queue_.pop();
  1734. return true;
  1735. }
  1736. return false;
  1737. }
  1738. case TASK_FLUSH:
  1739. DCHECK_EQ(task.type, pending_flush_tasks_.front());
  1740. if ((codec_ != VideoCodec::kVP9 && reorder_queue_.size() == 0) ||
  1741. (codec_ == VideoCodec::kVP9 && output_queue_.empty())) {
  1742. DVLOG(1) << "Flush complete";
  1743. pending_flush_tasks_.pop();
  1744. client_->NotifyFlushDone();
  1745. task_queue_.pop();
  1746. return true;
  1747. }
  1748. return false;
  1749. case TASK_RESET:
  1750. DCHECK_EQ(task.type, pending_flush_tasks_.front());
  1751. if ((codec_ != VideoCodec::kVP9 && reorder_queue_.size() == 0) ||
  1752. (codec_ == VideoCodec::kVP9 && output_queue_.empty())) {
  1753. DVLOG(1) << "Reset complete";
  1754. waiting_for_idr_ = true;
  1755. pending_flush_tasks_.pop();
  1756. client_->NotifyResetDone();
  1757. task_queue_.pop();
  1758. return true;
  1759. }
  1760. return false;
  1761. case TASK_DESTROY:
  1762. NOTREACHED() << "Can't destroy while in STATE_DECODING";
  1763. NotifyError(ILLEGAL_STATE, SFT_PLATFORM_ERROR);
  1764. return false;
  1765. }
  1766. }
  1767. bool VTVideoDecodeAccelerator::ProcessReorderQueue() {
  1768. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1769. DCHECK_EQ(state_, STATE_DECODING);
  1770. if (reorder_queue_.empty())
  1771. return false;
  1772. // If the next task is a flush (because there is a pending flush or because
  1773. // the next frame is an IDR), then we don't need a full reorder buffer to send
  1774. // the next frame.
  1775. bool flushing =
  1776. !task_queue_.empty() && (task_queue_.front().type != TASK_FRAME ||
  1777. task_queue_.front().frame->is_idr ||
  1778. task_queue_.front().frame->has_mmco5);
  1779. size_t reorder_window = std::max(0, reorder_queue_.top()->reorder_window);
  1780. DVLOG(3) << __func__ << " size=" << reorder_queue_.size()
  1781. << " window=" << reorder_window << " flushing=" << flushing;
  1782. if (flushing || reorder_queue_.size() > reorder_window) {
  1783. if (ProcessFrame(*reorder_queue_.top())) {
  1784. reorder_queue_.pop();
  1785. return true;
  1786. }
  1787. }
  1788. return false;
  1789. }
  1790. bool VTVideoDecodeAccelerator::ProcessOutputQueue() {
  1791. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1792. DCHECK_EQ(state_, STATE_DECODING);
  1793. if (output_queue_.empty())
  1794. return false;
  1795. if (ProcessFrame(*output_queue_.front())) {
  1796. output_queue_.pop_front();
  1797. return true;
  1798. }
  1799. return false;
  1800. }
  1801. bool VTVideoDecodeAccelerator::ProcessFrame(const Frame& frame) {
  1802. DVLOG(3) << __func__ << "(" << frame.bitstream_id << ")";
  1803. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1804. DCHECK_EQ(state_, STATE_DECODING);
  1805. // If the next pending flush is for a reset, then the frame will be dropped.
  1806. bool resetting = !pending_flush_tasks_.empty() &&
  1807. pending_flush_tasks_.front() == TASK_RESET;
  1808. if (resetting)
  1809. return true;
  1810. DCHECK(frame.image.get());
  1811. // If the |image_size| has changed, request new picture buffers and then
  1812. // wait for them.
  1813. //
  1814. // TODO(sandersd): When used by GpuVideoDecoder, we don't need to bother
  1815. // with this. We can tell that is the case when we also have a timestamp.
  1816. if (picture_size_ != frame.image_size) {
  1817. // Dismiss current pictures.
  1818. for (int32_t picture_id : assigned_picture_ids_) {
  1819. DVLOG(3) << "DismissPictureBuffer(" << picture_id << ")";
  1820. client_->DismissPictureBuffer(picture_id);
  1821. }
  1822. assigned_picture_ids_.clear();
  1823. picture_info_map_.clear();
  1824. available_picture_ids_.clear();
  1825. // Request new pictures.
  1826. picture_size_ = frame.image_size;
  1827. // TODO(https://crbug.com/1210994): Remove RGBAF16 support, and expose only
  1828. // PIXEL_FORMAT_NV12 and PIXEL_FORMAT_YUV420P10.
  1829. picture_format_ = PIXEL_FORMAT_RGBAF16;
  1830. if (base::FeatureList::IsEnabled(kMultiPlaneVideoToolboxSharedImages)) {
  1831. // TODO(https://crbug.com/1233228): The UV planes of P010 frames cannot
  1832. // be represented in the current gfx::BufferFormat.
  1833. if (config_.profile != VP9PROFILE_PROFILE2 &&
  1834. config_.profile != HEVCPROFILE_MAIN10 &&
  1835. config_.profile != HEVCPROFILE_REXT)
  1836. picture_format_ = PIXEL_FORMAT_NV12;
  1837. }
  1838. DVLOG(3) << "ProvidePictureBuffers(" << kNumPictureBuffers
  1839. << frame.image_size.ToString() << ")";
  1840. client_->ProvidePictureBuffers(kNumPictureBuffers, picture_format_, 1,
  1841. frame.image_size,
  1842. gpu::GetPlatformSpecificTextureTarget());
  1843. return false;
  1844. }
  1845. return SendFrame(frame);
  1846. }
  1847. bool VTVideoDecodeAccelerator::SendFrame(const Frame& frame) {
  1848. DVLOG(2) << __func__ << "(" << frame.bitstream_id << ")";
  1849. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  1850. DCHECK_EQ(state_, STATE_DECODING);
  1851. DCHECK(frame.image.get());
  1852. if (available_picture_ids_.empty())
  1853. return false;
  1854. int32_t picture_id = available_picture_ids_.back();
  1855. auto it = picture_info_map_.find(picture_id);
  1856. DCHECK(it != picture_info_map_.end());
  1857. PictureInfo* picture_info = it->second.get();
  1858. DCHECK(picture_info->gl_images.empty());
  1859. const gfx::BufferFormat buffer_format =
  1860. config_.profile == VP9PROFILE_PROFILE2 ||
  1861. config_.profile == HEVCPROFILE_MAIN10 ||
  1862. config_.profile == HEVCPROFILE_REXT
  1863. ? gfx::BufferFormat::P010
  1864. : gfx::BufferFormat::YUV_420_BIPLANAR;
  1865. gfx::ColorSpace color_space = GetImageBufferColorSpace(frame.image);
  1866. std::vector<gfx::BufferPlane> planes;
  1867. switch (picture_format_) {
  1868. case PIXEL_FORMAT_NV12:
  1869. case PIXEL_FORMAT_YUV420P10:
  1870. planes.push_back(gfx::BufferPlane::Y);
  1871. planes.push_back(gfx::BufferPlane::UV);
  1872. break;
  1873. case PIXEL_FORMAT_RGBAF16:
  1874. planes.push_back(gfx::BufferPlane::DEFAULT);
  1875. break;
  1876. default:
  1877. NOTREACHED();
  1878. break;
  1879. }
  1880. for (size_t plane = 0; plane < planes.size(); ++plane) {
  1881. const gfx::Size plane_size(
  1882. CVPixelBufferGetWidthOfPlane(frame.image.get(), plane),
  1883. CVPixelBufferGetHeightOfPlane(frame.image.get(), plane));
  1884. gfx::BufferFormat plane_buffer_format =
  1885. gpu::GetPlaneBufferFormat(planes[plane], buffer_format);
  1886. // TODO(https://crbug.com/1108909): BGRA is not an appropriate value for
  1887. // these parameters.
  1888. const viz::ResourceFormat viz_resource_format =
  1889. (picture_format_ == PIXEL_FORMAT_RGBAF16)
  1890. ? viz::ResourceFormat::RGBA_F16
  1891. : viz::GetResourceFormat(plane_buffer_format);
  1892. const GLenum gl_format = viz::GLDataFormat(viz_resource_format);
  1893. scoped_refptr<gl::GLImageIOSurface> gl_image(
  1894. gl::GLImageIOSurface::Create(plane_size, gl_format));
  1895. if (!gl_image->InitializeWithCVPixelBuffer(
  1896. frame.image.get(), plane,
  1897. gfx::GenericSharedMemoryId(g_cv_pixel_buffer_ids.GetNext()),
  1898. plane_buffer_format)) {
  1899. NOTIFY_STATUS("Failed to initialize GLImageIOSurface", PLATFORM_FAILURE,
  1900. SFT_PLATFORM_ERROR);
  1901. }
  1902. gl_image->DisableInUseByWindowServer();
  1903. gl_image->SetColorSpaceForYUVToRGBConversion(color_space);
  1904. gl_image->SetColorSpaceShallow(color_space);
  1905. if (picture_info->uses_shared_images) {
  1906. gpu::SharedImageStub* shared_image_stub = client_->GetSharedImageStub();
  1907. DCHECK(shared_image_stub);
  1908. const uint32_t shared_image_usage =
  1909. gpu::SHARED_IMAGE_USAGE_DISPLAY | gpu::SHARED_IMAGE_USAGE_SCANOUT;
  1910. gpu::Mailbox mailbox = gpu::Mailbox::GenerateForSharedImage();
  1911. gpu::GLTextureImageBackingHelper::InitializeGLTextureParams gl_params;
  1912. // ANGLE-on-Metal exposes IOSurfaces via GL_TEXTURE_2D. Be robust to that.
  1913. gl_params.target = gl_client_.supports_arb_texture_rectangle
  1914. ? GL_TEXTURE_RECTANGLE_ARB
  1915. : GL_TEXTURE_2D;
  1916. gl_params.internal_format = gl_format;
  1917. gl_params.format = gl_format;
  1918. gl_params.type = GL_UNSIGNED_BYTE;
  1919. gl_params.is_cleared = true;
  1920. // Making the GL context current before performing below shared image
  1921. // tasks.
  1922. // TODO(vikassoni) : Remove if making context current is not required.
  1923. if (!gl_client_.make_context_current.Run()) {
  1924. DLOG(ERROR) << "Failed to make context current";
  1925. NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
  1926. return false;
  1927. }
  1928. auto shared_image = std::make_unique<gpu::GLImageBacking>(
  1929. gl_image, mailbox, viz_resource_format, plane_size, color_space,
  1930. kTopLeft_GrSurfaceOrigin, kOpaque_SkAlphaType, shared_image_usage,
  1931. gl_params, gl_client_.is_passthrough);
  1932. const bool success = shared_image_stub->factory()->RegisterBacking(
  1933. std::move(shared_image), /*allow_legacy_mailbox=*/false);
  1934. if (!success) {
  1935. DLOG(ERROR) << "Failed to register shared image";
  1936. NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
  1937. return false;
  1938. }
  1939. // Wrap the destroy callback in a lambda that ensures that it be called on
  1940. // the appropriate thread.
  1941. auto destroy_shared_image_lambda =
  1942. [](gpu::SharedImageStub::SharedImageDestructionCallback callback,
  1943. scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
  1944. task_runner->PostTask(FROM_HERE, base::BindOnce(std::move(callback),
  1945. gpu::SyncToken()));
  1946. };
  1947. auto destroy_shared_image_callback = base::BindOnce(
  1948. destroy_shared_image_lambda,
  1949. shared_image_stub->GetSharedImageDestructionCallback(mailbox),
  1950. gpu_task_runner_);
  1951. picture_info->scoped_shared_images.push_back(
  1952. scoped_refptr<Picture::ScopedSharedImage>(
  1953. new Picture::ScopedSharedImage(
  1954. mailbox, gl_params.target,
  1955. std::move(destroy_shared_image_callback))));
  1956. } else {
  1957. if (!gl_client_.bind_image.Run(picture_info->client_texture_id,
  1958. gpu::GetPlatformSpecificTextureTarget(),
  1959. gl_image, false)) {
  1960. DLOG(ERROR) << "Failed to bind image";
  1961. NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
  1962. return false;
  1963. }
  1964. }
  1965. picture_info->gl_images.push_back(gl_image);
  1966. }
  1967. picture_info->bitstream_id = frame.bitstream_id;
  1968. available_picture_ids_.pop_back();
  1969. DVLOG(3) << "PictureReady(picture_id=" << picture_id << ", "
  1970. << "bitstream_id=" << frame.bitstream_id << ")";
  1971. Picture picture(picture_id, frame.bitstream_id, gfx::Rect(frame.image_size),
  1972. color_space, true);
  1973. // The GLImageIOSurface keeps the IOSurface alive as long as it exists, but
  1974. // bound textures do not, and they can outlive the GLImageIOSurface if they
  1975. // are deleted in the command buffer before they are used by the platform GL
  1976. // implementation. (https://crbug.com/930479#c69)
  1977. //
  1978. // A fence is required whenever a GLImage is bound, but we can't know in
  1979. // advance whether that will happen.
  1980. //
  1981. // TODO(sandersd): Can GLImageIOSurface be responsible for fences, so that
  1982. // we don't need to use them when the image is never bound? Bindings are
  1983. // typically only created when WebGL is in use.
  1984. picture.set_read_lock_fences_enabled(true);
  1985. for (size_t plane = 0; plane < planes.size(); ++plane) {
  1986. picture.set_scoped_shared_image(picture_info->scoped_shared_images[plane],
  1987. plane);
  1988. }
  1989. if (picture_format_ == PIXEL_FORMAT_NV12)
  1990. picture.set_is_webgpu_compatible(true);
  1991. client_->PictureReady(std::move(picture));
  1992. return true;
  1993. }
  1994. void VTVideoDecodeAccelerator::NotifyError(
  1995. Error vda_error_type,
  1996. VTVDASessionFailureType session_failure_type) {
  1997. DCHECK_LT(session_failure_type, SFT_MAX + 1);
  1998. if (!gpu_task_runner_->BelongsToCurrentThread()) {
  1999. gpu_task_runner_->PostTask(
  2000. FROM_HERE,
  2001. base::BindOnce(&VTVideoDecodeAccelerator::NotifyError, weak_this_,
  2002. vda_error_type, session_failure_type));
  2003. } else if (state_ == STATE_DECODING) {
  2004. state_ = STATE_ERROR;
  2005. UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
  2006. session_failure_type, SFT_MAX + 1);
  2007. client_->NotifyError(vda_error_type);
  2008. }
  2009. }
  2010. void VTVideoDecodeAccelerator::WriteToMediaLog(MediaLogMessageLevel level,
  2011. const std::string& message) {
  2012. if (!gpu_task_runner_->BelongsToCurrentThread()) {
  2013. gpu_task_runner_->PostTask(
  2014. FROM_HERE, base::BindOnce(&VTVideoDecodeAccelerator::WriteToMediaLog,
  2015. weak_this_, level, message));
  2016. return;
  2017. }
  2018. DVLOG(1) << __func__ << "(" << static_cast<int>(level) << ") " << message;
  2019. if (media_log_)
  2020. media_log_->AddMessage(level, message);
  2021. }
  2022. void VTVideoDecodeAccelerator::QueueFlush(TaskType type) {
  2023. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  2024. pending_flush_tasks_.push(type);
  2025. decoder_task_runner_->PostTask(
  2026. FROM_HERE, base::BindOnce(&VTVideoDecodeAccelerator::FlushTask,
  2027. decoder_weak_this_, type));
  2028. // If this is a new flush request, see if we can make progress.
  2029. if (pending_flush_tasks_.size() == 1)
  2030. ProcessWorkQueues();
  2031. }
  2032. void VTVideoDecodeAccelerator::Flush() {
  2033. DVLOG(1) << __func__;
  2034. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  2035. QueueFlush(TASK_FLUSH);
  2036. }
  2037. void VTVideoDecodeAccelerator::Reset() {
  2038. DVLOG(1) << __func__;
  2039. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  2040. QueueFlush(TASK_RESET);
  2041. }
  2042. void VTVideoDecodeAccelerator::Destroy() {
  2043. DVLOG(1) << __func__;
  2044. DCHECK(gpu_task_runner_->BelongsToCurrentThread());
  2045. // For a graceful shutdown, return assigned buffers and flush before
  2046. // destructing |this|.
  2047. for (int32_t bitstream_id : assigned_bitstream_ids_)
  2048. client_->NotifyEndOfBitstreamBuffer(bitstream_id);
  2049. assigned_bitstream_ids_.clear();
  2050. state_ = STATE_DESTROYING;
  2051. QueueFlush(TASK_DESTROY);
  2052. }
  2053. bool VTVideoDecodeAccelerator::TryToSetupDecodeOnSeparateThread(
  2054. const base::WeakPtr<Client>& decode_client,
  2055. const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) {
  2056. return false;
  2057. }
  2058. bool VTVideoDecodeAccelerator::SupportsSharedImagePictureBuffers() const {
  2059. return true;
  2060. }
  2061. VideoDecodeAccelerator::TextureAllocationMode
  2062. VTVideoDecodeAccelerator::GetSharedImageTextureAllocationMode() const {
  2063. return VideoDecodeAccelerator::TextureAllocationMode::
  2064. kDoNotAllocateGLTextures;
  2065. }
  2066. // static
  2067. VideoDecodeAccelerator::SupportedProfiles
  2068. VTVideoDecodeAccelerator::GetSupportedProfiles(
  2069. const gpu::GpuDriverBugWorkarounds& workarounds) {
  2070. SupportedProfiles profiles;
  2071. if (!InitializeVideoToolbox())
  2072. return profiles;
  2073. for (const auto& supported_profile : kSupportedProfiles) {
  2074. if (supported_profile == VP9PROFILE_PROFILE0 ||
  2075. supported_profile == VP9PROFILE_PROFILE2) {
  2076. if (workarounds.disable_accelerated_vp9_decode)
  2077. continue;
  2078. if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_VP9))
  2079. continue;
  2080. // Success! We have VP9 hardware decoding support.
  2081. }
  2082. if (supported_profile == HEVCPROFILE_MAIN ||
  2083. supported_profile == HEVCPROFILE_MAIN10 ||
  2084. supported_profile == HEVCPROFILE_MAIN_STILL_PICTURE ||
  2085. supported_profile == HEVCPROFILE_REXT) {
  2086. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  2087. if (!workarounds.disable_accelerated_hevc_decode &&
  2088. base::FeatureList::IsEnabled(kPlatformHEVCDecoderSupport)) {
  2089. if (__builtin_available(macOS 11.0, *)) {
  2090. // Success! We have HEVC hardware decoding (or software
  2091. // decoding if the hardware is not good enough) support too.
  2092. SupportedProfile profile;
  2093. profile.profile = supported_profile;
  2094. profile.min_resolution.SetSize(16, 16);
  2095. // max supported resolution -> 8k 👍
  2096. profile.max_resolution.SetSize(8192, 8192);
  2097. profiles.push_back(profile);
  2098. }
  2099. }
  2100. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  2101. continue;
  2102. }
  2103. SupportedProfile profile;
  2104. profile.profile = supported_profile;
  2105. profile.min_resolution.SetSize(16, 16);
  2106. profile.max_resolution.SetSize(4096, 4096);
  2107. profiles.push_back(profile);
  2108. }
  2109. return profiles;
  2110. }
  2111. } // namespace media