hdr_copier_layer.mm 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/metal_util/hdr_copier_layer.h"
  5. #include <CoreVideo/CVPixelBuffer.h>
  6. #include <Metal/Metal.h>
  7. #include <MetalKit/MetalKit.h>
  8. #include "base/mac/foundation_util.h"
  9. #include "base/mac/scoped_cftyperef.h"
  10. #include "base/mac/scoped_nsobject.h"
  11. #include "base/notreached.h"
  12. #include "base/strings/sys_string_conversions.h"
  13. #include "components/metal_util/device.h"
  14. #include "third_party/skia/modules/skcms/skcms.h"
  15. #include "ui/gfx/color_space.h"
  16. namespace {
  17. // Source of the shader to perform tonemapping. Note that the functions
  18. // ToLinearSRGB, ToLinearPQ, and ToLinearHLG are copy-pasted from the GLSL
  19. // shader source in gfx::ColorTransform.
  20. // TODO(https://crbug.com/1101041): Add non-identity tonemapping to the shader.
  21. const char* tonemapping_shader_source =
  22. "#include <metal_stdlib>\n"
  23. "#include <simd/simd.h>\n"
  24. "using metal::float2;\n"
  25. "using metal::float3;\n"
  26. "using metal::float3x3;\n"
  27. "using metal::float4;\n"
  28. "using metal::sampler;\n"
  29. "using metal::texture2d;\n"
  30. "using metal::abs;\n"
  31. "using metal::exp;\n"
  32. "using metal::max;\n"
  33. "using metal::pow;\n"
  34. "using metal::sign;\n"
  35. "\n"
  36. "typedef struct {\n"
  37. " float4 clipSpacePosition [[position]];\n"
  38. " float2 texcoord;\n"
  39. "} RasterizerData;\n"
  40. "\n"
  41. "float ToLinearSRGB(float v) {\n"
  42. " float abs_v = abs(v);\n"
  43. " float sgn_v = sign(v);\n"
  44. " if (abs_v < 0.0404482362771082f)\n"
  45. " return v/12.92f;\n"
  46. " else\n"
  47. " return sgn_v*pow((abs_v+0.055f)/1.055f, 2.4f);\n"
  48. "}\n"
  49. "\n"
  50. "float ToLinearPQ(float v) {\n"
  51. " v = max(0.0f, v);\n"
  52. " constexpr float m1 = (2610.0 / 4096.0) / 4.0;\n"
  53. " constexpr float m2 = (2523.0 / 4096.0) * 128.0;\n"
  54. " constexpr float c1 = 3424.0 / 4096.0;\n"
  55. " constexpr float c2 = (2413.0 / 4096.0) * 32.0;\n"
  56. " constexpr float c3 = (2392.0 / 4096.0) * 32.0;\n"
  57. " float p = pow(v, 1.f / m2);\n"
  58. " v = pow(max(p - c1, 0.f) / (c2 - c3 * p), 1.f / m1);\n"
  59. " float sdr_white_level = 100.f;\n"
  60. " v *= 10000.f / sdr_white_level;\n"
  61. " return v;\n"
  62. "}\n"
  63. "\n"
  64. "float ToLinearHLG(float v) {\n"
  65. " constexpr float a = 0.17883277;\n"
  66. " constexpr float b = 0.28466892;\n"
  67. " constexpr float c = 0.55991073;\n"
  68. " v = max(0.f, v);\n"
  69. " if (v <= 0.5f)\n"
  70. " return (v * 2.f) * (v * 2.f);\n"
  71. " return exp((v - c) / a) + b;\n"
  72. "}\n"
  73. "\n"
  74. "vertex RasterizerData vertexShader(\n"
  75. " uint vertexID [[vertex_id]],\n"
  76. " constant float2 *positions[[buffer(0)]]) {\n"
  77. " RasterizerData out;\n"
  78. " out.clipSpacePosition = vector_float4(0.f, 0.f, 0.f, 1.f);\n"
  79. " out.clipSpacePosition.x = 2.f * positions[vertexID].x - 1.f;\n"
  80. " out.clipSpacePosition.y = -2.f * positions[vertexID].y + 1.f;\n"
  81. " out.texcoord = positions[vertexID];\n"
  82. " return out;\n"
  83. "}\n"
  84. "\n"
  85. "float3 ToneMap(float3 v) {\n"
  86. " return v;\n"
  87. "}\n"
  88. "\n"
  89. "fragment float4 fragmentShader(RasterizerData in [[stage_in]],\n"
  90. " texture2d<float> t [[texture(0)]],\n"
  91. " constant float3x3& m [[buffer(0)]],\n"
  92. " constant uint32_t& f [[buffer(1)]]) {\n"
  93. " constexpr sampler s(metal::mag_filter::nearest,\n"
  94. " metal::min_filter::nearest);\n"
  95. " float4 color = t.sample(s, in.texcoord);\n"
  96. " switch (f) {\n"
  97. " case 1:\n"
  98. " color.x = ToLinearSRGB(color.x);\n"
  99. " color.y = ToLinearSRGB(color.y);\n"
  100. " color.z = ToLinearSRGB(color.z);\n"
  101. " break;\n"
  102. " case 2:\n"
  103. " color.x = ToLinearPQ(color.x);\n"
  104. " color.y = ToLinearPQ(color.y);\n"
  105. " color.z = ToLinearPQ(color.z);\n"
  106. " break;\n"
  107. " case 3:\n"
  108. " color.x = ToLinearHLG(color.x);\n"
  109. " color.y = ToLinearHLG(color.y);\n"
  110. " color.z = ToLinearHLG(color.z);\n"
  111. " break;\n"
  112. " default:\n"
  113. " break;\n"
  114. " }\n"
  115. " color.xyz = ToneMap(m * color.xyz);\n"
  116. " return color;\n"
  117. "}\n";
  118. // Return the integer to use to specify a transfer function to the shader
  119. // defined in the above source. Return 0 if the transfer function is
  120. // unsupported.
  121. uint32_t GetTransferFunctionIndex(const gfx::ColorSpace& color_space) {
  122. switch (color_space.GetTransferID()) {
  123. case gfx::ColorSpace::TransferID::SRGB_HDR:
  124. return 1;
  125. case gfx::ColorSpace::TransferID::PQ:
  126. return 2;
  127. case gfx::ColorSpace::TransferID::HLG:
  128. return 3;
  129. default:
  130. return 0;
  131. }
  132. }
  133. // Convert from an IOSurface's pixel format to a MTLPixelFormat. Crash on any
  134. // unsupported formats.
  135. MTLPixelFormat IOSurfaceGetMTLPixelFormat(IOSurfaceRef buffer) {
  136. uint32_t format = IOSurfaceGetPixelFormat(buffer);
  137. switch (format) {
  138. case kCVPixelFormatType_64RGBAHalf:
  139. return MTLPixelFormatRGBA16Float;
  140. case kCVPixelFormatType_ARGB2101010LEPacked:
  141. return MTLPixelFormatBGR10A2Unorm;
  142. case kCVPixelFormatType_32BGRA:
  143. return MTLPixelFormatBGRA8Unorm;
  144. case kCVPixelFormatType_32RGBA:
  145. return MTLPixelFormatRGBA8Unorm;
  146. default:
  147. break;
  148. }
  149. return MTLPixelFormatInvalid;
  150. }
  151. base::scoped_nsprotocol<id<MTLRenderPipelineState>> CreateRenderPipelineState(
  152. id<MTLDevice> device) {
  153. base::scoped_nsprotocol<id<MTLRenderPipelineState>> render_pipeline_state;
  154. base::scoped_nsprotocol<id<MTLLibrary>> library;
  155. {
  156. NSError* error = nil;
  157. base::scoped_nsobject<NSString> source([[NSString alloc]
  158. initWithCString:tonemapping_shader_source
  159. encoding:NSASCIIStringEncoding]);
  160. base::scoped_nsobject<MTLCompileOptions> options(
  161. [[MTLCompileOptions alloc] init]);
  162. library.reset([device newLibraryWithSource:source
  163. options:options
  164. error:&error]);
  165. if (error) {
  166. NSLog(@"Failed to compile shader: %@", error);
  167. return render_pipeline_state;
  168. }
  169. }
  170. {
  171. base::scoped_nsprotocol<id<MTLFunction>> vertex_function(
  172. [library newFunctionWithName:@"vertexShader"]);
  173. base::scoped_nsprotocol<id<MTLFunction>> fragment_function(
  174. [library newFunctionWithName:@"fragmentShader"]);
  175. NSError* error = nil;
  176. base::scoped_nsobject<MTLRenderPipelineDescriptor> desc(
  177. [[MTLRenderPipelineDescriptor alloc] init]);
  178. [desc setVertexFunction:vertex_function];
  179. [desc setFragmentFunction:fragment_function];
  180. [[desc colorAttachments][0] setPixelFormat:MTLPixelFormatRGBA16Float];
  181. render_pipeline_state.reset(
  182. [device newRenderPipelineStateWithDescriptor:desc error:&error]);
  183. if (error) {
  184. NSLog(@"Failed to create render pipeline state: %@", error);
  185. return render_pipeline_state;
  186. }
  187. }
  188. return render_pipeline_state;
  189. }
  190. } // namespace
  191. #if !defined(MAC_OS_X_VERSION_10_15)
  192. API_AVAILABLE(macos(10.15))
  193. @interface CAMetalLayer (Forward)
  194. @property(readonly) id<MTLDevice> preferredDevice;
  195. @end
  196. #endif
  197. API_AVAILABLE(macos(10.15))
  198. @interface HDRCopierLayer : CAMetalLayer {
  199. base::scoped_nsprotocol<id<MTLRenderPipelineState>> _render_pipeline_state;
  200. gfx::ColorSpace _color_space;
  201. }
  202. - (id)init;
  203. - (void)setHDRContents:(IOSurfaceRef)buffer
  204. withColorSpace:(gfx::ColorSpace)color_space;
  205. @end
  206. @implementation HDRCopierLayer
  207. - (id)init {
  208. if (self = [super init]) {
  209. base::scoped_nsprotocol<id<MTLDevice>> device(metal::CreateDefaultDevice());
  210. [self setWantsExtendedDynamicRangeContent:YES];
  211. [self setDevice:device];
  212. [self setOpaque:NO];
  213. [self setPresentsWithTransaction:YES];
  214. [self setPixelFormat:MTLPixelFormatRGBA16Float];
  215. [self setColorspace:CGColorSpaceCreateWithName(
  216. kCGColorSpaceExtendedLinearSRGB)];
  217. }
  218. return self;
  219. }
  220. - (void)setHDRContents:(IOSurfaceRef)buffer
  221. withColorSpace:(gfx::ColorSpace)color_space {
  222. // Retrieve information about the IOSurface.
  223. size_t width = IOSurfaceGetWidth(buffer);
  224. size_t height = IOSurfaceGetHeight(buffer);
  225. MTLPixelFormat mtl_format = IOSurfaceGetMTLPixelFormat(buffer);
  226. if (mtl_format == MTLPixelFormatInvalid) {
  227. DLOG(ERROR) << "Unsupported IOSurface format.";
  228. return;
  229. }
  230. // Set metadata for tone mapping.
  231. if (_color_space != color_space) {
  232. CAEDRMetadata* edr_metadata = nil;
  233. switch (color_space.GetTransferID()) {
  234. case gfx::ColorSpace::TransferID::PQ:
  235. edr_metadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0
  236. maxLuminance:10000
  237. opticalOutputScale:100];
  238. break;
  239. case gfx::ColorSpace::TransferID::HLG:
  240. edr_metadata = [CAEDRMetadata HLGMetadata];
  241. break;
  242. default:
  243. [self setEDRMetadata:nil];
  244. break;
  245. }
  246. [self setEDRMetadata:edr_metadata];
  247. _color_space = color_space;
  248. }
  249. // Migrate to the MTLDevice on which the CAMetalLayer is being composited, if
  250. // known.
  251. if ([self respondsToSelector:@selector(preferredDevice)]) {
  252. id<MTLDevice> preferred_device = [self preferredDevice];
  253. if (preferred_device)
  254. [self setDevice:preferred_device];
  255. }
  256. id<MTLDevice> device = [self device];
  257. // When the device changes, rebuild the RenderPipelineState.
  258. if (device != [_render_pipeline_state device])
  259. _render_pipeline_state = CreateRenderPipelineState(device);
  260. if (!_render_pipeline_state)
  261. return;
  262. // Update the layer's properties to match the IOSurface.
  263. [self setDrawableSize:CGSizeMake(width, height)];
  264. // Create a texture to wrap the IOSurface.
  265. base::scoped_nsprotocol<id<MTLTexture>> buffer_texture;
  266. {
  267. base::scoped_nsobject<MTLTextureDescriptor> tex_desc(
  268. [MTLTextureDescriptor new]);
  269. [tex_desc setTextureType:MTLTextureType2D];
  270. [tex_desc setUsage:MTLTextureUsageShaderRead];
  271. [tex_desc setPixelFormat:mtl_format];
  272. [tex_desc setWidth:width];
  273. [tex_desc setHeight:height];
  274. [tex_desc setDepth:1];
  275. [tex_desc setMipmapLevelCount:1];
  276. [tex_desc setArrayLength:1];
  277. [tex_desc setSampleCount:1];
  278. [tex_desc setStorageMode:MTLStorageModeManaged];
  279. buffer_texture.reset([device newTextureWithDescriptor:tex_desc
  280. iosurface:buffer
  281. plane:0]);
  282. }
  283. // Create a texture to wrap the drawable.
  284. id<CAMetalDrawable> drawable = [self nextDrawable];
  285. id<MTLTexture> drawable_texture = [drawable texture];
  286. // Copy from the IOSurface to the drawable.
  287. base::scoped_nsprotocol<id<MTLCommandQueue>> command_queue(
  288. [device newCommandQueue]);
  289. id<MTLCommandBuffer> command_buffer = [command_queue commandBuffer];
  290. id<MTLRenderCommandEncoder> encoder = nil;
  291. {
  292. MTLRenderPassDescriptor* desc =
  293. [MTLRenderPassDescriptor renderPassDescriptor];
  294. desc.colorAttachments[0].texture = drawable_texture;
  295. desc.colorAttachments[0].loadAction = MTLLoadActionClear;
  296. desc.colorAttachments[0].storeAction = MTLStoreActionStore;
  297. desc.colorAttachments[0].clearColor = MTLClearColorMake(0.0, 0.0, 0.0, 0.0);
  298. encoder = [command_buffer renderCommandEncoderWithDescriptor:desc];
  299. MTLViewport viewport;
  300. viewport.originX = 0;
  301. viewport.originY = 0;
  302. viewport.width = width;
  303. viewport.height = height;
  304. viewport.znear = -1.0;
  305. viewport.zfar = 1.0;
  306. [encoder setViewport:viewport];
  307. [encoder setRenderPipelineState:_render_pipeline_state];
  308. [encoder setFragmentTexture:buffer_texture atIndex:0];
  309. }
  310. {
  311. simd::float2 positions[6] = {
  312. simd::make_float2(0, 0), simd::make_float2(0, 1),
  313. simd::make_float2(1, 1), simd::make_float2(1, 1),
  314. simd::make_float2(1, 0), simd::make_float2(0, 0),
  315. };
  316. // The value of |transfer_function| corresponds to the value as used in
  317. // the above shader source.
  318. uint32_t transfer_function_index = GetTransferFunctionIndex(color_space);
  319. DCHECK(transfer_function_index);
  320. // Matrix is the primary transform matrix from |color_space| to sRGB.
  321. simd::float3x3 matrix;
  322. {
  323. skcms_Matrix3x3 src_to_xyz;
  324. skcms_Matrix3x3 srgb_to_xyz;
  325. skcms_Matrix3x3 xyz_to_srgb;
  326. color_space.GetPrimaryMatrix(&src_to_xyz);
  327. gfx::ColorSpace::CreateSRGB().GetPrimaryMatrix(&srgb_to_xyz);
  328. skcms_Matrix3x3_invert(&srgb_to_xyz, &xyz_to_srgb);
  329. skcms_Matrix3x3 m = skcms_Matrix3x3_concat(&xyz_to_srgb, &src_to_xyz);
  330. matrix = simd::float3x3(
  331. simd::make_float3(m.vals[0][0], m.vals[1][0], m.vals[2][0]),
  332. simd::make_float3(m.vals[0][1], m.vals[1][1], m.vals[2][1]),
  333. simd::make_float3(m.vals[0][2], m.vals[1][2], m.vals[2][2]));
  334. }
  335. [encoder setFragmentBytes:&transfer_function_index
  336. length:sizeof(transfer_function_index)
  337. atIndex:1];
  338. [encoder setVertexBytes:positions length:sizeof(positions) atIndex:0];
  339. [encoder setFragmentBytes:&matrix length:sizeof(matrix) atIndex:0];
  340. [encoder drawPrimitives:MTLPrimitiveTypeTriangle
  341. vertexStart:0
  342. vertexCount:6];
  343. }
  344. [encoder endEncoding];
  345. [command_buffer commit];
  346. [command_buffer waitUntilScheduled];
  347. [drawable present];
  348. }
  349. @end
  350. namespace metal {
  351. CALayer* CreateHDRCopierLayer() {
  352. // If this is hit by non-10.15 paths (e.g, for testing), then return an
  353. // ordinary CALayer. Calling setContents on that CALayer will work fine
  354. // (HDR content will be clipped, but that would have happened anyway).
  355. if (@available(macos 10.15, *))
  356. return [[HDRCopierLayer alloc] init];
  357. NOTREACHED();
  358. return nil;
  359. }
  360. void UpdateHDRCopierLayer(CALayer* layer,
  361. IOSurfaceRef buffer,
  362. const gfx::ColorSpace& color_space) {
  363. if (@available(macos 10.15, *)) {
  364. if (auto* hdr_copier_layer = base::mac::ObjCCast<HDRCopierLayer>(layer)) {
  365. [hdr_copier_layer setHDRContents:buffer withColorSpace:color_space];
  366. return;
  367. }
  368. }
  369. NOTREACHED();
  370. }
  371. bool ShouldUseHDRCopier(IOSurfaceRef buffer,
  372. const gfx::ColorSpace& color_space) {
  373. if (@available(macos 10.15, *)) {
  374. return GetTransferFunctionIndex(color_space) &&
  375. IOSurfaceGetMTLPixelFormat(buffer) != MTLPixelFormatInvalid;
  376. }
  377. return false;
  378. }
  379. } // namespace metal