text_detection_impl_win.cc 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "services/shape_detection/text_detection_impl_win.h"
  5. #include <windows.foundation.collections.h>
  6. #include <windows.globalization.h>
  7. #include <memory>
  8. #include <string>
  9. #include "base/bind.h"
  10. #include "base/logging.h"
  11. #include "base/win/core_winrt_util.h"
  12. #include "base/win/post_async_results.h"
  13. #include "base/win/scoped_hstring.h"
  14. #include "base/win/windows_version.h"
  15. #include "mojo/public/cpp/bindings/pending_receiver.h"
  16. #include "mojo/public/cpp/bindings/self_owned_receiver.h"
  17. #include "services/shape_detection/detection_utils_win.h"
  18. #include "services/shape_detection/text_detection_impl.h"
  19. #include "ui/gfx/geometry/rect_f.h"
  20. namespace shape_detection {
  21. using ABI::Windows::Foundation::IAsyncOperation;
  22. using ABI::Windows::Foundation::Collections::IVectorView;
  23. using ABI::Windows::Globalization::ILanguageFactory;
  24. using ABI::Windows::Graphics::Imaging::ISoftwareBitmap;
  25. using ABI::Windows::Graphics::Imaging::ISoftwareBitmapStatics;
  26. using ABI::Windows::Media::Ocr::IOcrEngine;
  27. using ABI::Windows::Media::Ocr::IOcrEngineStatics;
  28. using ABI::Windows::Media::Ocr::IOcrLine;
  29. using ABI::Windows::Media::Ocr::IOcrResult;
  30. using ABI::Windows::Media::Ocr::IOcrWord;
  31. using ABI::Windows::Media::Ocr::OcrLine;
  32. using ABI::Windows::Media::Ocr::OcrResult;
  33. using ABI::Windows::Media::Ocr::OcrWord;
  34. using base::win::GetActivationFactory;
  35. using base::win::ScopedHString;
  36. using Microsoft::WRL::ComPtr;
  37. // static
  38. void TextDetectionImpl::Create(
  39. mojo::PendingReceiver<mojom::TextDetection> receiver) {
  40. // OcrEngine class is only available in Win 10 onwards (v10.0.10240.0) that
  41. // documents in
  42. // https://docs.microsoft.com/en-us/uwp/api/windows.media.ocr.ocrengine.
  43. if (base::win::GetVersion() < base::win::Version::WIN10) {
  44. DVLOG(1) << "Optical character recognition not supported before Windows 10";
  45. return;
  46. }
  47. DCHECK_GE(base::win::OSInfo::GetInstance()->version_number().build, 10240u);
  48. // Loads functions dynamically at runtime to prevent library dependencies.
  49. if (!(base::win::ResolveCoreWinRTDelayload() &&
  50. ScopedHString::ResolveCoreWinRTStringDelayload())) {
  51. DLOG(ERROR) << "Failed loading functions from combase.dll";
  52. return;
  53. }
  54. // Text Detection specification only supports Latin-1 text as documented in
  55. // https://wicg.github.io/shape-detection-api/text.html#text-detection-api.
  56. // TODO(junwei.fu): https://crbug.com/794097 consider supporting other Latin
  57. // script language.
  58. ScopedHString language_hstring = ScopedHString::Create("en");
  59. if (!language_hstring.is_valid())
  60. return;
  61. ComPtr<ILanguageFactory> language_factory;
  62. HRESULT hr =
  63. GetActivationFactory<ILanguageFactory,
  64. RuntimeClass_Windows_Globalization_Language>(
  65. &language_factory);
  66. if (FAILED(hr)) {
  67. DLOG(ERROR) << "ILanguage factory failed: "
  68. << logging::SystemErrorCodeToString(hr);
  69. return;
  70. }
  71. ComPtr<ABI::Windows::Globalization::ILanguage> language;
  72. hr = language_factory->CreateLanguage(language_hstring.get(), &language);
  73. if (FAILED(hr)) {
  74. DLOG(ERROR) << "Create language failed: "
  75. << logging::SystemErrorCodeToString(hr);
  76. return;
  77. }
  78. ComPtr<IOcrEngineStatics> engine_factory;
  79. hr = GetActivationFactory<IOcrEngineStatics,
  80. RuntimeClass_Windows_Media_Ocr_OcrEngine>(
  81. &engine_factory);
  82. if (FAILED(hr)) {
  83. DLOG(ERROR) << "IOcrEngineStatics factory failed: "
  84. << logging::SystemErrorCodeToString(hr);
  85. return;
  86. }
  87. boolean is_supported = false;
  88. hr = engine_factory->IsLanguageSupported(language.Get(), &is_supported);
  89. if (FAILED(hr) || !is_supported)
  90. return;
  91. ComPtr<IOcrEngine> ocr_engine;
  92. hr = engine_factory->TryCreateFromLanguage(language.Get(), &ocr_engine);
  93. if (FAILED(hr)) {
  94. DLOG(ERROR) << "Create engine failed from language: "
  95. << logging::SystemErrorCodeToString(hr);
  96. return;
  97. }
  98. ComPtr<ISoftwareBitmapStatics> bitmap_factory;
  99. hr = GetActivationFactory<
  100. ISoftwareBitmapStatics,
  101. RuntimeClass_Windows_Graphics_Imaging_SoftwareBitmap>(&bitmap_factory);
  102. if (FAILED(hr)) {
  103. DLOG(ERROR) << "ISoftwareBitmapStatics factory failed: "
  104. << logging::SystemErrorCodeToString(hr);
  105. return;
  106. }
  107. auto impl = std::make_unique<TextDetectionImplWin>(std::move(ocr_engine),
  108. std::move(bitmap_factory));
  109. auto* impl_ptr = impl.get();
  110. impl_ptr->SetReceiver(
  111. mojo::MakeSelfOwnedReceiver(std::move(impl), std::move(receiver)));
  112. }
  113. TextDetectionImplWin::TextDetectionImplWin(
  114. ComPtr<IOcrEngine> ocr_engine,
  115. ComPtr<ISoftwareBitmapStatics> bitmap_factory)
  116. : ocr_engine_(std::move(ocr_engine)),
  117. bitmap_factory_(std::move(bitmap_factory)) {
  118. DCHECK(ocr_engine_);
  119. DCHECK(bitmap_factory_);
  120. }
  121. TextDetectionImplWin::~TextDetectionImplWin() = default;
  122. void TextDetectionImplWin::Detect(const SkBitmap& bitmap,
  123. DetectCallback callback) {
  124. if (FAILED(BeginDetect(bitmap))) {
  125. // No detection taking place; run |callback| with an empty array of results.
  126. std::move(callback).Run(std::vector<mojom::TextDetectionResultPtr>());
  127. return;
  128. }
  129. // Hold on the callback until AsyncOperation completes.
  130. recognize_text_callback_ = std::move(callback);
  131. // This prevents the Detect function from being called before the
  132. // AsyncOperation completes.
  133. receiver_->PauseIncomingMethodCallProcessing();
  134. }
  135. HRESULT TextDetectionImplWin::BeginDetect(const SkBitmap& bitmap) {
  136. ComPtr<ISoftwareBitmap> win_bitmap =
  137. CreateWinBitmapFromSkBitmap(bitmap, bitmap_factory_.Get());
  138. if (!win_bitmap)
  139. return E_FAIL;
  140. // Recognize text asynchronously.
  141. ComPtr<IAsyncOperation<OcrResult*>> async_op;
  142. const HRESULT hr = ocr_engine_->RecognizeAsync(win_bitmap.Get(), &async_op);
  143. if (FAILED(hr)) {
  144. DLOG(ERROR) << "Recognize text asynchronously failed: "
  145. << logging::SystemErrorCodeToString(hr);
  146. return hr;
  147. }
  148. // Use WeakPtr to bind the callback so that the once callback will not be run
  149. // if this object has been already destroyed. |win_bitmap| needs to be kept
  150. // alive until OnTextDetected().
  151. return base::win::PostAsyncResults(
  152. std::move(async_op),
  153. base::BindOnce(&TextDetectionImplWin::OnTextDetected,
  154. weak_factory_.GetWeakPtr(), std::move(win_bitmap)));
  155. }
  156. std::vector<mojom::TextDetectionResultPtr>
  157. TextDetectionImplWin::BuildTextDetectionResult(ComPtr<IOcrResult> ocr_result) {
  158. std::vector<mojom::TextDetectionResultPtr> results;
  159. if (!ocr_result)
  160. return results;
  161. ComPtr<IVectorView<OcrLine*>> ocr_lines;
  162. HRESULT hr = ocr_result->get_Lines(&ocr_lines);
  163. if (FAILED(hr)) {
  164. DLOG(ERROR) << "Get Lines failed: " << logging::SystemErrorCodeToString(hr);
  165. return results;
  166. }
  167. uint32_t count;
  168. hr = ocr_lines->get_Size(&count);
  169. if (FAILED(hr)) {
  170. DLOG(ERROR) << "get_Size failed: " << logging::SystemErrorCodeToString(hr);
  171. return results;
  172. }
  173. results.reserve(count);
  174. for (uint32_t i = 0; i < count; ++i) {
  175. ComPtr<IOcrLine> line;
  176. hr = ocr_lines->GetAt(i, &line);
  177. if (FAILED(hr))
  178. break;
  179. HSTRING text;
  180. hr = line->get_Text(&text);
  181. if (FAILED(hr))
  182. break;
  183. // Gets bounding box with the words detected in the current line of Text.
  184. ComPtr<IVectorView<OcrWord*>> ocr_words;
  185. hr = line->get_Words(&ocr_words);
  186. if (FAILED(hr))
  187. break;
  188. uint32_t words_count;
  189. hr = ocr_words->get_Size(&words_count);
  190. if (FAILED(hr))
  191. break;
  192. auto result = shape_detection::mojom::TextDetectionResult::New();
  193. for (uint32_t word_num = 0; word_num < words_count; ++word_num) {
  194. ComPtr<IOcrWord> word;
  195. hr = ocr_words->GetAt(word_num, &word);
  196. if (FAILED(hr))
  197. break;
  198. ABI::Windows::Foundation::Rect bounds;
  199. hr = word->get_BoundingRect(&bounds);
  200. if (FAILED(hr))
  201. break;
  202. result->bounding_box = gfx::UnionRects(
  203. result->bounding_box,
  204. gfx::RectF(bounds.X, bounds.Y, bounds.Width, bounds.Height));
  205. }
  206. result->raw_value = ScopedHString(text).GetAsUTF8();
  207. results.push_back(std::move(result));
  208. }
  209. return results;
  210. }
  211. // |win_bitmap| is passed here so that it is kept alive until the AsyncOperation
  212. // completes because RecognizeAsync does not hold a reference.
  213. void TextDetectionImplWin::OnTextDetected(
  214. ComPtr<ISoftwareBitmap> /* win_bitmap */,
  215. ComPtr<IOcrResult> ocr_result) {
  216. std::move(recognize_text_callback_)
  217. .Run(BuildTextDetectionResult(std::move(ocr_result)));
  218. receiver_->ResumeIncomingMethodCallProcessing();
  219. }
  220. } // namespace shape_detection