123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254 |
- // Copyright 2017 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "services/shape_detection/text_detection_impl_win.h"
- #include <windows.foundation.collections.h>
- #include <windows.globalization.h>
- #include <memory>
- #include <string>
- #include "base/bind.h"
- #include "base/logging.h"
- #include "base/win/core_winrt_util.h"
- #include "base/win/post_async_results.h"
- #include "base/win/scoped_hstring.h"
- #include "base/win/windows_version.h"
- #include "mojo/public/cpp/bindings/pending_receiver.h"
- #include "mojo/public/cpp/bindings/self_owned_receiver.h"
- #include "services/shape_detection/detection_utils_win.h"
- #include "services/shape_detection/text_detection_impl.h"
- #include "ui/gfx/geometry/rect_f.h"
- namespace shape_detection {
- using ABI::Windows::Foundation::IAsyncOperation;
- using ABI::Windows::Foundation::Collections::IVectorView;
- using ABI::Windows::Globalization::ILanguageFactory;
- using ABI::Windows::Graphics::Imaging::ISoftwareBitmap;
- using ABI::Windows::Graphics::Imaging::ISoftwareBitmapStatics;
- using ABI::Windows::Media::Ocr::IOcrEngine;
- using ABI::Windows::Media::Ocr::IOcrEngineStatics;
- using ABI::Windows::Media::Ocr::IOcrLine;
- using ABI::Windows::Media::Ocr::IOcrResult;
- using ABI::Windows::Media::Ocr::IOcrWord;
- using ABI::Windows::Media::Ocr::OcrLine;
- using ABI::Windows::Media::Ocr::OcrResult;
- using ABI::Windows::Media::Ocr::OcrWord;
- using base::win::GetActivationFactory;
- using base::win::ScopedHString;
- using Microsoft::WRL::ComPtr;
- // static
- void TextDetectionImpl::Create(
- mojo::PendingReceiver<mojom::TextDetection> receiver) {
- // OcrEngine class is only available in Win 10 onwards (v10.0.10240.0) that
- // documents in
- // https://docs.microsoft.com/en-us/uwp/api/windows.media.ocr.ocrengine.
- if (base::win::GetVersion() < base::win::Version::WIN10) {
- DVLOG(1) << "Optical character recognition not supported before Windows 10";
- return;
- }
- DCHECK_GE(base::win::OSInfo::GetInstance()->version_number().build, 10240u);
- // Loads functions dynamically at runtime to prevent library dependencies.
- if (!(base::win::ResolveCoreWinRTDelayload() &&
- ScopedHString::ResolveCoreWinRTStringDelayload())) {
- DLOG(ERROR) << "Failed loading functions from combase.dll";
- return;
- }
- // Text Detection specification only supports Latin-1 text as documented in
- // https://wicg.github.io/shape-detection-api/text.html#text-detection-api.
- // TODO(junwei.fu): https://crbug.com/794097 consider supporting other Latin
- // script language.
- ScopedHString language_hstring = ScopedHString::Create("en");
- if (!language_hstring.is_valid())
- return;
- ComPtr<ILanguageFactory> language_factory;
- HRESULT hr =
- GetActivationFactory<ILanguageFactory,
- RuntimeClass_Windows_Globalization_Language>(
- &language_factory);
- if (FAILED(hr)) {
- DLOG(ERROR) << "ILanguage factory failed: "
- << logging::SystemErrorCodeToString(hr);
- return;
- }
- ComPtr<ABI::Windows::Globalization::ILanguage> language;
- hr = language_factory->CreateLanguage(language_hstring.get(), &language);
- if (FAILED(hr)) {
- DLOG(ERROR) << "Create language failed: "
- << logging::SystemErrorCodeToString(hr);
- return;
- }
- ComPtr<IOcrEngineStatics> engine_factory;
- hr = GetActivationFactory<IOcrEngineStatics,
- RuntimeClass_Windows_Media_Ocr_OcrEngine>(
- &engine_factory);
- if (FAILED(hr)) {
- DLOG(ERROR) << "IOcrEngineStatics factory failed: "
- << logging::SystemErrorCodeToString(hr);
- return;
- }
- boolean is_supported = false;
- hr = engine_factory->IsLanguageSupported(language.Get(), &is_supported);
- if (FAILED(hr) || !is_supported)
- return;
- ComPtr<IOcrEngine> ocr_engine;
- hr = engine_factory->TryCreateFromLanguage(language.Get(), &ocr_engine);
- if (FAILED(hr)) {
- DLOG(ERROR) << "Create engine failed from language: "
- << logging::SystemErrorCodeToString(hr);
- return;
- }
- ComPtr<ISoftwareBitmapStatics> bitmap_factory;
- hr = GetActivationFactory<
- ISoftwareBitmapStatics,
- RuntimeClass_Windows_Graphics_Imaging_SoftwareBitmap>(&bitmap_factory);
- if (FAILED(hr)) {
- DLOG(ERROR) << "ISoftwareBitmapStatics factory failed: "
- << logging::SystemErrorCodeToString(hr);
- return;
- }
- auto impl = std::make_unique<TextDetectionImplWin>(std::move(ocr_engine),
- std::move(bitmap_factory));
- auto* impl_ptr = impl.get();
- impl_ptr->SetReceiver(
- mojo::MakeSelfOwnedReceiver(std::move(impl), std::move(receiver)));
- }
- TextDetectionImplWin::TextDetectionImplWin(
- ComPtr<IOcrEngine> ocr_engine,
- ComPtr<ISoftwareBitmapStatics> bitmap_factory)
- : ocr_engine_(std::move(ocr_engine)),
- bitmap_factory_(std::move(bitmap_factory)) {
- DCHECK(ocr_engine_);
- DCHECK(bitmap_factory_);
- }
- TextDetectionImplWin::~TextDetectionImplWin() = default;
- void TextDetectionImplWin::Detect(const SkBitmap& bitmap,
- DetectCallback callback) {
- if (FAILED(BeginDetect(bitmap))) {
- // No detection taking place; run |callback| with an empty array of results.
- std::move(callback).Run(std::vector<mojom::TextDetectionResultPtr>());
- return;
- }
- // Hold on the callback until AsyncOperation completes.
- recognize_text_callback_ = std::move(callback);
- // This prevents the Detect function from being called before the
- // AsyncOperation completes.
- receiver_->PauseIncomingMethodCallProcessing();
- }
- HRESULT TextDetectionImplWin::BeginDetect(const SkBitmap& bitmap) {
- ComPtr<ISoftwareBitmap> win_bitmap =
- CreateWinBitmapFromSkBitmap(bitmap, bitmap_factory_.Get());
- if (!win_bitmap)
- return E_FAIL;
- // Recognize text asynchronously.
- ComPtr<IAsyncOperation<OcrResult*>> async_op;
- const HRESULT hr = ocr_engine_->RecognizeAsync(win_bitmap.Get(), &async_op);
- if (FAILED(hr)) {
- DLOG(ERROR) << "Recognize text asynchronously failed: "
- << logging::SystemErrorCodeToString(hr);
- return hr;
- }
- // Use WeakPtr to bind the callback so that the once callback will not be run
- // if this object has been already destroyed. |win_bitmap| needs to be kept
- // alive until OnTextDetected().
- return base::win::PostAsyncResults(
- std::move(async_op),
- base::BindOnce(&TextDetectionImplWin::OnTextDetected,
- weak_factory_.GetWeakPtr(), std::move(win_bitmap)));
- }
- std::vector<mojom::TextDetectionResultPtr>
- TextDetectionImplWin::BuildTextDetectionResult(ComPtr<IOcrResult> ocr_result) {
- std::vector<mojom::TextDetectionResultPtr> results;
- if (!ocr_result)
- return results;
- ComPtr<IVectorView<OcrLine*>> ocr_lines;
- HRESULT hr = ocr_result->get_Lines(&ocr_lines);
- if (FAILED(hr)) {
- DLOG(ERROR) << "Get Lines failed: " << logging::SystemErrorCodeToString(hr);
- return results;
- }
- uint32_t count;
- hr = ocr_lines->get_Size(&count);
- if (FAILED(hr)) {
- DLOG(ERROR) << "get_Size failed: " << logging::SystemErrorCodeToString(hr);
- return results;
- }
- results.reserve(count);
- for (uint32_t i = 0; i < count; ++i) {
- ComPtr<IOcrLine> line;
- hr = ocr_lines->GetAt(i, &line);
- if (FAILED(hr))
- break;
- HSTRING text;
- hr = line->get_Text(&text);
- if (FAILED(hr))
- break;
- // Gets bounding box with the words detected in the current line of Text.
- ComPtr<IVectorView<OcrWord*>> ocr_words;
- hr = line->get_Words(&ocr_words);
- if (FAILED(hr))
- break;
- uint32_t words_count;
- hr = ocr_words->get_Size(&words_count);
- if (FAILED(hr))
- break;
- auto result = shape_detection::mojom::TextDetectionResult::New();
- for (uint32_t word_num = 0; word_num < words_count; ++word_num) {
- ComPtr<IOcrWord> word;
- hr = ocr_words->GetAt(word_num, &word);
- if (FAILED(hr))
- break;
- ABI::Windows::Foundation::Rect bounds;
- hr = word->get_BoundingRect(&bounds);
- if (FAILED(hr))
- break;
- result->bounding_box = gfx::UnionRects(
- result->bounding_box,
- gfx::RectF(bounds.X, bounds.Y, bounds.Width, bounds.Height));
- }
- result->raw_value = ScopedHString(text).GetAsUTF8();
- results.push_back(std::move(result));
- }
- return results;
- }
- // |win_bitmap| is passed here so that it is kept alive until the AsyncOperation
- // completes because RecognizeAsync does not hold a reference.
- void TextDetectionImplWin::OnTextDetected(
- ComPtr<ISoftwareBitmap> /* win_bitmap */,
- ComPtr<IOcrResult> ocr_result) {
- std::move(recognize_text_callback_)
- .Run(BuildTextDetectionResult(std::move(ocr_result)));
- receiver_->ResumeIncomingMethodCallProcessing();
- }
- } // namespace shape_detection
|