gl_image_io_surface.mm 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ui/gl/gl_image_io_surface.h"
  5. #include <map>
  6. #include "base/bind.h"
  7. #include "base/callback_helpers.h"
  8. #include "base/mac/foundation_util.h"
  9. #include "base/metrics/histogram_macros.h"
  10. #include "base/trace_event/memory_allocator_dump.h"
  11. #include "base/trace_event/memory_dump_manager.h"
  12. #include "base/trace_event/process_memory_dump.h"
  13. #include "base/trace_event/trace_event.h"
  14. #include "ui/gfx/buffer_format_util.h"
  15. #include "ui/gfx/mac/display_icc_profiles.h"
  16. #include "ui/gfx/mac/io_surface.h"
  17. #include "ui/gl/buffer_format_utils.h"
  18. #include "ui/gl/gl_bindings.h"
  19. #include "ui/gl/gl_context.h"
  20. #include "ui/gl/gl_enums.h"
  21. #include "ui/gl/gl_version_info.h"
  22. #include "ui/gl/scoped_binders.h"
  23. #include "ui/gl/yuv_to_rgb_converter.h"
  24. #if defined(USE_EGL)
  25. #include "ui/gl/gl_image_io_surface_egl.h"
  26. #include "ui/gl/gl_implementation.h"
  27. #endif // defined(USE_EGL)
  28. // Note that this must be included after gl_bindings.h to avoid conflicts.
  29. #include <OpenGL/CGLIOSurface.h>
  30. #include <Quartz/Quartz.h>
  31. #include <stddef.h>
  32. using gfx::BufferFormat;
  33. namespace gl {
  34. namespace {
  35. bool ValidInternalFormat(unsigned internalformat) {
  36. switch (internalformat) {
  37. case GL_RED:
  38. case GL_R16_EXT:
  39. case GL_RG:
  40. case GL_BGRA_EXT:
  41. case GL_RGB:
  42. case GL_RGB10_A2_EXT:
  43. case GL_RGB_YCBCR_420V_CHROMIUM:
  44. case GL_RGB_YCBCR_422_CHROMIUM:
  45. case GL_RGB_YCBCR_P010_CHROMIUM:
  46. case GL_RGBA:
  47. return true;
  48. default:
  49. return false;
  50. }
  51. }
  52. GLenum TextureFormat(gfx::BufferFormat format) {
  53. switch (format) {
  54. case gfx::BufferFormat::R_8:
  55. return GL_RED;
  56. case gfx::BufferFormat::R_16:
  57. return GL_R16_EXT;
  58. case gfx::BufferFormat::RG_88:
  59. return GL_RG;
  60. case gfx::BufferFormat::RG_1616:
  61. return GL_RG16_EXT;
  62. case gfx::BufferFormat::BGRA_8888:
  63. case gfx::BufferFormat::BGRX_8888: // See https://crbug.com/595948.
  64. case gfx::BufferFormat::RGBA_8888:
  65. case gfx::BufferFormat::RGBX_8888:
  66. case gfx::BufferFormat::RGBA_F16:
  67. case gfx::BufferFormat::BGRA_1010102:
  68. return GL_RGBA;
  69. case gfx::BufferFormat::YUV_420_BIPLANAR:
  70. return GL_RGB_YCBCR_420V_CHROMIUM;
  71. case gfx::BufferFormat::P010:
  72. return GL_RGB_YCBCR_P010_CHROMIUM;
  73. case gfx::BufferFormat::BGR_565:
  74. case gfx::BufferFormat::RGBA_4444:
  75. case gfx::BufferFormat::RGBA_1010102:
  76. case gfx::BufferFormat::YVU_420:
  77. NOTREACHED() << gfx::BufferFormatToString(format);
  78. return 0;
  79. }
  80. NOTREACHED();
  81. return 0;
  82. }
  83. GLenum DataFormat(gfx::BufferFormat format) {
  84. switch (format) {
  85. case gfx::BufferFormat::R_8:
  86. return GL_RED;
  87. case gfx::BufferFormat::R_16:
  88. return GL_R16_EXT;
  89. case gfx::BufferFormat::RG_88:
  90. return GL_RG;
  91. case gfx::BufferFormat::RG_1616:
  92. return GL_RG16_EXT;
  93. case gfx::BufferFormat::BGRA_8888:
  94. case gfx::BufferFormat::BGRX_8888:
  95. case gfx::BufferFormat::RGBA_8888: // See https://crbug.com/533677#c6.
  96. case gfx::BufferFormat::BGRA_1010102:
  97. return GL_BGRA;
  98. case gfx::BufferFormat::RGBA_F16:
  99. return GL_RGBA;
  100. case gfx::BufferFormat::BGR_565:
  101. case gfx::BufferFormat::RGBA_4444:
  102. case gfx::BufferFormat::RGBX_8888:
  103. case gfx::BufferFormat::RGBA_1010102:
  104. case gfx::BufferFormat::YVU_420:
  105. case gfx::BufferFormat::YUV_420_BIPLANAR:
  106. case gfx::BufferFormat::P010:
  107. NOTREACHED() << gfx::BufferFormatToString(format);
  108. return 0;
  109. }
  110. NOTREACHED();
  111. return 0;
  112. }
  113. GLenum DataType(gfx::BufferFormat format) {
  114. switch (format) {
  115. case gfx::BufferFormat::R_8:
  116. case gfx::BufferFormat::RG_88:
  117. return GL_UNSIGNED_BYTE;
  118. case gfx::BufferFormat::R_16:
  119. case gfx::BufferFormat::RG_1616:
  120. return GL_UNSIGNED_SHORT;
  121. case gfx::BufferFormat::BGRA_8888:
  122. case gfx::BufferFormat::BGRX_8888:
  123. case gfx::BufferFormat::RGBA_8888:
  124. return GL_UNSIGNED_INT_8_8_8_8_REV;
  125. case gfx::BufferFormat::BGRA_1010102:
  126. return GL_UNSIGNED_INT_2_10_10_10_REV;
  127. case gfx::BufferFormat::RGBA_F16:
  128. return GL_HALF_APPLE;
  129. case gfx::BufferFormat::BGR_565:
  130. case gfx::BufferFormat::RGBA_4444:
  131. case gfx::BufferFormat::RGBX_8888:
  132. case gfx::BufferFormat::RGBA_1010102:
  133. case gfx::BufferFormat::YVU_420:
  134. case gfx::BufferFormat::YUV_420_BIPLANAR:
  135. case gfx::BufferFormat::P010:
  136. NOTREACHED() << gfx::BufferFormatToString(format);
  137. return 0;
  138. }
  139. NOTREACHED();
  140. return 0;
  141. }
  142. // When an IOSurface is bound to a texture with internalformat "GL_RGB", many
  143. // OpenGL operations are broken. Therefore, don't allow an IOSurface to be bound
  144. // with GL_RGB unless overridden via BindTexImageWithInternalformat.
  145. // https://crbug.com/595948, https://crbug.com/699566.
  146. GLenum ConvertRequestedInternalFormat(GLenum internalformat) {
  147. if (internalformat == GL_RGB)
  148. return GL_RGBA;
  149. return internalformat;
  150. }
  151. } // namespace
  152. // static
  153. GLImageIOSurface* GLImageIOSurface::Create(const gfx::Size& size,
  154. unsigned internalformat) {
  155. #if defined(USE_EGL)
  156. switch (GetGLImplementation()) {
  157. case kGLImplementationEGLGLES2:
  158. case kGLImplementationEGLANGLE:
  159. return new GLImageIOSurfaceEGL(size, internalformat);
  160. default:
  161. break;
  162. }
  163. #endif // defined(USE_EGL)
  164. return new GLImageIOSurface(size, internalformat);
  165. }
  166. GLImageIOSurface::GLImageIOSurface(const gfx::Size& size,
  167. unsigned internalformat)
  168. : size_(size),
  169. internalformat_(ConvertRequestedInternalFormat(internalformat)),
  170. client_internalformat_(internalformat),
  171. format_(gfx::BufferFormat::RGBA_8888) {}
  172. GLImageIOSurface::~GLImageIOSurface() {
  173. DCHECK(thread_checker_.CalledOnValidThread());
  174. }
  175. bool GLImageIOSurface::Initialize(IOSurfaceRef io_surface,
  176. uint32_t io_surface_plane,
  177. gfx::GenericSharedMemoryId io_surface_id,
  178. gfx::BufferFormat format) {
  179. DCHECK(thread_checker_.CalledOnValidThread());
  180. DCHECK(!io_surface_);
  181. if (!io_surface) {
  182. LOG(ERROR) << "Invalid IOSurface";
  183. return false;
  184. }
  185. if (!ValidInternalFormat(internalformat_)) {
  186. LOG(ERROR) << "Invalid internalformat: "
  187. << GLEnums::GetStringEnum(internalformat_);
  188. return false;
  189. }
  190. if (!ValidFormat(format)) {
  191. LOG(ERROR) << "Invalid format: " << gfx::BufferFormatToString(format);
  192. return false;
  193. }
  194. format_ = format;
  195. io_surface_.reset(io_surface, base::scoped_policy::RETAIN);
  196. io_surface_id_ = io_surface_id;
  197. // YUV_420_BIPLANAR and P010 are not supported by BindTexImage. CopyTexImage
  198. // is supported by these formats as that performs conversion to RGB as part of
  199. // the copy operation.
  200. if (format_ != gfx::BufferFormat::YUV_420_BIPLANAR &&
  201. format_ != gfx::BufferFormat::P010) {
  202. io_surface_plane_ = io_surface_plane;
  203. }
  204. return true;
  205. }
  206. bool GLImageIOSurface::InitializeWithCVPixelBuffer(
  207. CVPixelBufferRef cv_pixel_buffer,
  208. uint32_t io_surface_plane,
  209. gfx::GenericSharedMemoryId io_surface_id,
  210. gfx::BufferFormat format) {
  211. IOSurfaceRef io_surface = CVPixelBufferGetIOSurface(cv_pixel_buffer);
  212. if (!io_surface) {
  213. LOG(ERROR) << "Can't init GLImage from CVPixelBuffer with no IOSurface";
  214. return false;
  215. }
  216. if (!Initialize(io_surface, io_surface_plane, io_surface_id, format))
  217. return false;
  218. cv_pixel_buffer_.reset(cv_pixel_buffer, base::scoped_policy::RETAIN);
  219. return true;
  220. }
  221. gfx::Size GLImageIOSurface::GetSize() {
  222. return size_;
  223. }
  224. unsigned GLImageIOSurface::GetInternalFormat() {
  225. return internalformat_;
  226. }
  227. unsigned GLImageIOSurface::GetDataType() {
  228. return gl::BufferFormatToGLDataType(format_);
  229. }
  230. GLImageIOSurface::BindOrCopy GLImageIOSurface::ShouldBindOrCopy() {
  231. return io_surface_plane_ == kInvalidIOSurfacePlane ? COPY : BIND;
  232. }
  233. bool GLImageIOSurface::BindTexImage(unsigned target) {
  234. return BindTexImageWithInternalformat(target, 0);
  235. }
  236. bool GLImageIOSurface::BindTexImageWithInternalformat(unsigned target,
  237. unsigned internalformat) {
  238. DCHECK(thread_checker_.CalledOnValidThread());
  239. DCHECK_EQ(BIND, ShouldBindOrCopy());
  240. TRACE_EVENT0("gpu", "GLImageIOSurface::BindTexImage");
  241. base::TimeTicks start_time = base::TimeTicks::Now();
  242. DCHECK(io_surface_);
  243. if (!BindTexImageImpl(target, internalformat)) {
  244. return false;
  245. }
  246. UMA_HISTOGRAM_TIMES("GPU.IOSurface.TexImageTime",
  247. base::TimeTicks::Now() - start_time);
  248. return true;
  249. }
  250. bool GLImageIOSurface::BindTexImageImpl(unsigned target,
  251. unsigned internalformat) {
  252. if (target != GL_TEXTURE_RECTANGLE_ARB) {
  253. // This might be supported in the future. For now, perform strict
  254. // validation so we know what's going on.
  255. LOG(ERROR) << "IOSurface requires TEXTURE_RECTANGLE_ARB target";
  256. return false;
  257. }
  258. CGLContextObj cgl_context =
  259. static_cast<CGLContextObj>(GLContext::GetCurrent()->GetHandle());
  260. GLenum texture_format =
  261. internalformat ? internalformat : TextureFormat(format_);
  262. CGLError cgl_error = CGLTexImageIOSurface2D(
  263. cgl_context, GL_TEXTURE_RECTANGLE_ARB, texture_format, size_.width(),
  264. size_.height(), DataFormat(format_), DataType(format_), io_surface_.get(),
  265. io_surface_plane_);
  266. if (cgl_error != kCGLNoError) {
  267. LOG(ERROR) << "Error in CGLTexImageIOSurface2D: "
  268. << CGLErrorString(cgl_error);
  269. return false;
  270. }
  271. return true;
  272. }
  273. bool GLImageIOSurface::CopyTexImage(unsigned target) {
  274. DCHECK(thread_checker_.CalledOnValidThread());
  275. DCHECK_EQ(COPY, ShouldBindOrCopy());
  276. GLContext* gl_context = GLContext::GetCurrent();
  277. DCHECK(gl_context);
  278. YUVToRGBConverter* yuv_to_rgb_converter =
  279. gl_context->GetYUVToRGBConverter(color_space_for_yuv_to_rgb_);
  280. DCHECK(yuv_to_rgb_converter);
  281. // Note that state restoration is done explicitly instead of scoped binders to
  282. // avoid https://crbug.com/601729.
  283. GLint rgb_texture = 0;
  284. GLenum target_getter = 0;
  285. switch (target) {
  286. case GL_TEXTURE_2D:
  287. target_getter = GL_TEXTURE_BINDING_2D;
  288. break;
  289. case GL_TEXTURE_CUBE_MAP:
  290. target_getter = GL_TEXTURE_BINDING_CUBE_MAP;
  291. break;
  292. case GL_TEXTURE_EXTERNAL_OES:
  293. target_getter = GL_TEXTURE_BINDING_EXTERNAL_OES;
  294. break;
  295. case GL_TEXTURE_RECTANGLE_ARB:
  296. target_getter = GL_TEXTURE_BINDING_RECTANGLE_ARB;
  297. break;
  298. default:
  299. NOTIMPLEMENTED() << " Target not supported.";
  300. return false;
  301. }
  302. glGetIntegerv(target_getter, &rgb_texture);
  303. base::ScopedClosureRunner destroy_resources_runner(
  304. base::BindOnce(base::RetainBlock(^{
  305. glBindTexture(target, rgb_texture);
  306. })));
  307. const auto src_type =
  308. format_ == gfx::BufferFormat::P010 ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE;
  309. CGLContextObj cgl_context = CGLGetCurrentContext();
  310. {
  311. glBindTexture(GL_TEXTURE_RECTANGLE_ARB, yuv_to_rgb_converter->y_texture());
  312. CGLError cgl_error = CGLTexImageIOSurface2D(
  313. cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RED, size_.width(),
  314. size_.height(), GL_RED, src_type, io_surface_, 0);
  315. if (cgl_error != kCGLNoError) {
  316. LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the Y plane. "
  317. << cgl_error;
  318. return false;
  319. }
  320. }
  321. {
  322. glBindTexture(GL_TEXTURE_RECTANGLE_ARB, yuv_to_rgb_converter->uv_texture());
  323. CGLError cgl_error = CGLTexImageIOSurface2D(
  324. cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RG, size_.width() / 2,
  325. size_.height() / 2, GL_RG, src_type, io_surface_, 1);
  326. if (cgl_error != kCGLNoError) {
  327. LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the UV plane. "
  328. << cgl_error;
  329. return false;
  330. }
  331. }
  332. yuv_to_rgb_converter->CopyYUV420ToRGB(target, size_, rgb_texture, src_type);
  333. return true;
  334. }
  335. bool GLImageIOSurface::CopyTexSubImage(unsigned target,
  336. const gfx::Point& offset,
  337. const gfx::Rect& rect) {
  338. return false;
  339. }
  340. void GLImageIOSurface::OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd,
  341. uint64_t process_tracing_id,
  342. const std::string& dump_name) {
  343. size_t size_bytes = 0;
  344. if (io_surface_) {
  345. if (io_surface_plane_ == kInvalidIOSurfacePlane) {
  346. size_bytes = IOSurfaceGetAllocSize(io_surface_);
  347. } else {
  348. size_bytes =
  349. IOSurfaceGetBytesPerRowOfPlane(io_surface_, io_surface_plane_) *
  350. IOSurfaceGetHeightOfPlane(io_surface_, io_surface_plane_);
  351. }
  352. }
  353. base::trace_event::MemoryAllocatorDump* dump =
  354. pmd->CreateAllocatorDump(dump_name);
  355. dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
  356. base::trace_event::MemoryAllocatorDump::kUnitsBytes,
  357. static_cast<uint64_t>(size_bytes));
  358. // The process tracing id is to identify the GpuMemoryBuffer client that
  359. // created the allocation. For CVPixelBufferRefs, there is no corresponding
  360. // GpuMemoryBuffer, so use an invalid process id.
  361. if (cv_pixel_buffer_) {
  362. process_tracing_id =
  363. base::trace_event::MemoryDumpManager::kInvalidTracingProcessId;
  364. }
  365. // Create an edge using the GMB GenericSharedMemoryId if the image is not
  366. // anonymous. Otherwise, add another nested node to account for the anonymous
  367. // IOSurface.
  368. if (io_surface_id_.is_valid()) {
  369. auto guid = GetGenericSharedGpuMemoryGUIDForTracing(process_tracing_id,
  370. io_surface_id_);
  371. pmd->CreateSharedGlobalAllocatorDump(guid);
  372. pmd->AddOwnershipEdge(dump->guid(), guid);
  373. } else {
  374. std::string anonymous_dump_name = dump_name + "/anonymous-iosurface";
  375. base::trace_event::MemoryAllocatorDump* anonymous_dump =
  376. pmd->CreateAllocatorDump(anonymous_dump_name);
  377. anonymous_dump->AddScalar(
  378. base::trace_event::MemoryAllocatorDump::kNameSize,
  379. base::trace_event::MemoryAllocatorDump::kUnitsBytes,
  380. static_cast<uint64_t>(size_bytes));
  381. anonymous_dump->AddScalar("width", "pixels", size_.width());
  382. anonymous_dump->AddScalar("height", "pixels", size_.height());
  383. }
  384. }
  385. bool GLImageIOSurface::EmulatingRGB() const {
  386. return client_internalformat_ == GL_RGB;
  387. }
  388. bool GLImageIOSurface::IsInUseByWindowServer() const {
  389. // IOSurfaceIsInUse() will always return true if the IOSurface is wrapped in
  390. // a CVPixelBuffer. Ignore the signal for such IOSurfaces (which are the ones
  391. // output by hardware video decode).
  392. if (disable_in_use_by_window_server_)
  393. return false;
  394. return IOSurfaceIsInUse(io_surface_.get());
  395. }
  396. void GLImageIOSurface::DisableInUseByWindowServer() {
  397. disable_in_use_by_window_server_ = true;
  398. }
  399. void GLImageIOSurface::SetColorSpaceForYUVToRGBConversion(
  400. const gfx::ColorSpace& color_space) {
  401. DCHECK(color_space.IsValid());
  402. DCHECK_NE(color_space, color_space.GetAsFullRangeRGB());
  403. color_space_for_yuv_to_rgb_ = color_space;
  404. }
  405. void GLImageIOSurface::SetColorSpaceShallow(
  406. const gfx::ColorSpace& color_space) {
  407. GLImage::SetColorSpace(color_space);
  408. }
  409. base::ScopedCFTypeRef<IOSurfaceRef> GLImageIOSurface::io_surface() {
  410. return io_surface_;
  411. }
  412. base::ScopedCFTypeRef<CVPixelBufferRef> GLImageIOSurface::cv_pixel_buffer() {
  413. return cv_pixel_buffer_;
  414. }
  415. GLImage::Type GLImageIOSurface::GetType() const {
  416. return Type::IOSURFACE;
  417. }
  418. void GLImageIOSurface::SetColorSpace(const gfx::ColorSpace& color_space) {
  419. if (color_space_ == color_space)
  420. return;
  421. GLImage::SetColorSpace(color_space);
  422. // Prefer to use data from DisplayICCProfiles, which will give a byte-for-byte
  423. // match for color spaces of the system displays. Note that DisplayICCProfiles
  424. // is not used in IOSurfaceSetColorSpace because that call may be made in the
  425. // renderer process (e.g, for video frames).
  426. base::ScopedCFTypeRef<CFDataRef> cf_data =
  427. gfx::DisplayICCProfiles::GetInstance()->GetDataForColorSpace(color_space);
  428. if (cf_data) {
  429. IOSurfaceSetValue(io_surface_, CFSTR("IOSurfaceColorSpace"), cf_data);
  430. return;
  431. }
  432. // Only if that fails, fall back to IOSurfaceSetColorSpace, which will
  433. // generate a profile.
  434. IOSurfaceSetColorSpace(io_surface_, color_space);
  435. }
  436. // static
  437. unsigned GLImageIOSurface::GetInternalFormatForTesting(
  438. gfx::BufferFormat format) {
  439. DCHECK(ValidFormat(format));
  440. return TextureFormat(format);
  441. }
  442. // static
  443. GLImageIOSurface* GLImageIOSurface::FromGLImage(GLImage* image) {
  444. if (!image || image->GetType() != Type::IOSURFACE)
  445. return nullptr;
  446. return static_cast<GLImageIOSurface*>(image);
  447. }
  448. // static
  449. bool GLImageIOSurface::ValidFormat(gfx::BufferFormat format) {
  450. switch (format) {
  451. case gfx::BufferFormat::R_8:
  452. case gfx::BufferFormat::RG_88:
  453. case gfx::BufferFormat::R_16:
  454. case gfx::BufferFormat::RG_1616:
  455. case gfx::BufferFormat::BGRA_8888:
  456. case gfx::BufferFormat::BGRX_8888:
  457. case gfx::BufferFormat::RGBA_8888:
  458. case gfx::BufferFormat::RGBX_8888:
  459. case gfx::BufferFormat::RGBA_F16:
  460. case gfx::BufferFormat::BGRA_1010102:
  461. case gfx::BufferFormat::YUV_420_BIPLANAR:
  462. case gfx::BufferFormat::P010:
  463. return true;
  464. case gfx::BufferFormat::BGR_565:
  465. case gfx::BufferFormat::RGBA_4444:
  466. case gfx::BufferFormat::RGBA_1010102:
  467. case gfx::BufferFormat::YVU_420:
  468. return false;
  469. }
  470. NOTREACHED();
  471. return false;
  472. }
  473. } // namespace gl