annotator.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef SERVICES_IMAGE_ANNOTATION_ANNOTATOR_H_
  5. #define SERVICES_IMAGE_ANNOTATION_ANNOTATOR_H_
  6. #include <deque>
  7. #include <list>
  8. #include <map>
  9. #include <memory>
  10. #include <set>
  11. #include <string>
  12. #include <utility>
  13. #include "base/gtest_prod_util.h"
  14. #include "base/memory/scoped_refptr.h"
  15. #include "base/memory/weak_ptr.h"
  16. #include "base/time/time.h"
  17. #include "base/timer/timer.h"
  18. #include "mojo/public/cpp/bindings/pending_receiver.h"
  19. #include "mojo/public/cpp/bindings/pending_remote.h"
  20. #include "mojo/public/cpp/bindings/receiver_set.h"
  21. #include "mojo/public/cpp/bindings/remote.h"
  22. #include "services/data_decoder/public/mojom/json_parser.mojom.h"
  23. #include "services/image_annotation/public/mojom/image_annotation.mojom.h"
  24. #include "services/network/public/cpp/shared_url_loader_factory.h"
  25. #include "services/network/public/cpp/simple_url_loader.h"
  26. #include "url/gurl.h"
  27. namespace image_annotation {
  28. // The annotator communicates with the external image annotation server to
  29. // perform image labeling at the behest of clients.
  30. //
  31. // Clients make requests of the annotator by providing an image "source ID"
  32. // (which is either an image URL or hash of an image data URI) and an associated
  33. // ImageProcessor (the interface through which the annotator can obtain image
  34. // pixels if necessary).
  35. //
  36. // The annotator maintains a cache of previously-computed results, and will
  37. // compute new results either by sending image URLs (for publicly-crawled
  38. // images) or image pixels to the external server.
  39. class Annotator : public mojom::Annotator {
  40. public:
  41. class Client {
  42. public:
  43. virtual ~Client() {}
  44. virtual void BindJsonParser(
  45. mojo::PendingReceiver<data_decoder::mojom::JsonParser> receiver) = 0;
  46. virtual std::vector<std::string> GetAcceptLanguages() = 0;
  47. virtual std::vector<std::string> GetTopLanguages() = 0;
  48. virtual void RecordLanguageMetrics(
  49. const std::string& page_language,
  50. const std::string& requested_language) = 0;
  51. };
  52. // The HTTP request header in which the API key should be transmitted.
  53. static constexpr char kGoogApiKeyHeader[] = "X-Goog-Api-Key";
  54. // The minimum side length needed to request description annotations.
  55. static constexpr int32_t kDescMinDimension = 150;
  56. // The maximum aspect ratio permitted to request description annotations.
  57. static constexpr double kDescMaxAspectRatio = 2.5;
  58. // The minimum side length needed to request icon annotations.
  59. static constexpr int32_t kIconMinDimension = 16;
  60. // The maximum side length needed to request icon annotations.
  61. static constexpr int32_t kIconMaxDimension = 256;
  62. // The maximum aspect ratio permitted to request icon annotations.
  63. // (Most icons are square, but something like an ellipsis / "more" menu
  64. // can have a long aspect ratio.)
  65. static constexpr double kIconMaxAspectRatio = 5.0;
  66. // Constructs an annotator.
  67. // |pixels_server_url| : the URL to use when the annotator sends image
  68. // pixel data to get back annotations. The
  69. // annotator gracefully handles (i.e. returns
  70. // errors when constructed with) an empty server URL.
  71. // |langs_server_url| : the URL to use when the annotator requests the
  72. // set of languages supported by the server.
  73. // |api_key| : the Google API key used to authenticate
  74. // communication with the image annotation server. If
  75. // empty, no API key header will be sent.
  76. // |throttle| : the miminum amount of time to wait between sending
  77. // new HTTP requests to the image annotation server.
  78. // |batch_size| : The maximum number of image annotation requests that
  79. // should be batched into a single request to the
  80. // server.
  81. // |min_ocr_confidence|: The minimum confidence value needed to return an OCR
  82. // result.
  83. Annotator(GURL pixels_server_url,
  84. GURL langs_server_url,
  85. std::string api_key,
  86. base::TimeDelta throttle,
  87. int batch_size,
  88. double min_ocr_confidence,
  89. scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
  90. std::unique_ptr<Client> client);
  91. Annotator(const Annotator&) = delete;
  92. Annotator& operator=(const Annotator&) = delete;
  93. ~Annotator() override;
  94. // Start providing behavior for the given Mojo receiver.
  95. void BindReceiver(mojo::PendingReceiver<mojom::Annotator> receiver);
  96. // mojom::Annotator:
  97. void AnnotateImage(const std::string& source_id,
  98. const std::string& description_language_tag,
  99. mojo::PendingRemote<mojom::ImageProcessor> image,
  100. AnnotateImageCallback callback) override;
  101. private:
  102. FRIEND_TEST_ALL_PREFIXES(AnnotatorTest, DescLanguage);
  103. FRIEND_TEST_ALL_PREFIXES(AnnotatorTest, ComputePreferredLanguage);
  104. FRIEND_TEST_ALL_PREFIXES(AnnotatorTest, FetchServerLanguages);
  105. FRIEND_TEST_ALL_PREFIXES(AnnotatorTest, ServerLanguagesMustContainEnglish);
  106. FRIEND_TEST_ALL_PREFIXES(AnnotatorTest, LanguageFallback);
  107. // The relevant info for a request from a client feature for a single image.
  108. struct ClientRequestInfo {
  109. ClientRequestInfo(
  110. mojo::PendingRemote<mojom::ImageProcessor> image_processor,
  111. AnnotateImageCallback callback);
  112. ~ClientRequestInfo();
  113. mojo::Remote<mojom::ImageProcessor>
  114. image_processor; // The interface to use for local
  115. // processing for this client.
  116. AnnotateImageCallback callback; // The callback to execute when
  117. // processing has finished.
  118. };
  119. // The relevant info for a request to the image annotation server for a single
  120. // image.
  121. struct ServerRequestInfo {
  122. ServerRequestInfo(const std::string& source_id,
  123. bool desc_requested,
  124. bool icon_requested,
  125. const std::string& desc_lang_tag,
  126. const std::vector<uint8_t>& image_bytes);
  127. ServerRequestInfo(const ServerRequestInfo& other) = delete;
  128. ~ServerRequestInfo();
  129. // Use in a deque requires a move-assign operator.
  130. ServerRequestInfo& operator=(ServerRequestInfo&& other);
  131. ServerRequestInfo& operator=(const ServerRequestInfo& other) = delete;
  132. std::string source_id; // The URL or hashed data URI for the image.
  133. bool desc_requested; // Whether or not descriptions have been requested.
  134. bool icon_requested; // Whether or not icons have been requested.
  135. std::string desc_lang_tag; // The language in which descriptions have been
  136. // requested.
  137. std::vector<uint8_t> image_bytes; // The encoded pixel data for the image.
  138. };
  139. // The pair of (source ID, desc lang) for a client request.
  140. //
  141. // Unique request keys represent unique requests to the server (i.e. two
  142. // client requests that induce the same request key can be served by a single
  143. // server request).
  144. using RequestKey = std::pair<std::string, std::string>;
  145. // List of URL loader objects.
  146. using UrlLoaderList = std::list<std::unique_ptr<network::SimpleURLLoader>>;
  147. // Returns true if the given dimensions fit the policy of the description
  148. // backend (i.e. the image has size / shape on which it is acceptable to run
  149. // the description model).
  150. static bool IsWithinDescPolicy(int32_t width, int32_t height);
  151. // Returns true if the given dimensions fit the policy of the icon
  152. // backend (i.e. the image has size / shape on which it is acceptable to run
  153. // the icon model).
  154. static bool IsWithinIconPolicy(int32_t width, int32_t height);
  155. // Constructs and returns a JSON object containing an request for the
  156. // given images.
  157. static std::string FormatJsonRequest(
  158. std::deque<ServerRequestInfo>::iterator begin_it,
  159. std::deque<ServerRequestInfo>::iterator end_it);
  160. // Creates a URL loader for an image annotation request.
  161. static std::unique_ptr<network::SimpleURLLoader> MakeRequestLoader(
  162. const GURL& server_url,
  163. const std::string& api_key);
  164. // Create or reuse a connection to the data decoder service for safe JSON
  165. // parsing.
  166. data_decoder::mojom::JsonParser* GetJsonParser();
  167. // Removes the given request, reassigning local processing if its associated
  168. // image processor had some ongoing.
  169. void RemoveRequestInfo(const RequestKey& request_key,
  170. std::list<ClientRequestInfo>::iterator request_info_it,
  171. bool canceled);
  172. // Called when a local handler returns compressed image data for the given
  173. // request key.
  174. void OnJpgImageDataReceived(
  175. const RequestKey& request_key,
  176. std::list<ClientRequestInfo>::iterator request_info_it,
  177. const std::vector<uint8_t>& image_bytes,
  178. int32_t width,
  179. int32_t height);
  180. // Called periodically to send the next batch of requests to the image
  181. // annotation server.
  182. void SendRequestBatchToServer();
  183. // Called when the image annotation server responds with annotations for the
  184. // given request keys.
  185. void OnServerResponseReceived(const std::set<RequestKey>& request_keys,
  186. UrlLoaderList::iterator server_request_it,
  187. std::unique_ptr<std::string> json_response);
  188. // Called when the data decoder service provides parsed JSON data for a server
  189. // response.
  190. void OnResponseJsonParsed(const std::set<RequestKey>& request_keys,
  191. absl::optional<base::Value> json_data,
  192. const absl::optional<std::string>& error);
  193. // Adds the given results to the cache (if successful) and notifies clients.
  194. void ProcessResults(
  195. const std::set<RequestKey>& request_keys,
  196. const std::map<std::string, mojom::AnnotateImageResultPtr>& results);
  197. std::string ComputePreferredLanguage(const std::string& page_lang) const;
  198. // Fetch the set of languages that the server supports.
  199. void FetchServerLanguages();
  200. // Handle the reply with the server languages.
  201. void OnServerLangsResponseReceived(
  202. const std::unique_ptr<std::string> json_response);
  203. // Parse the JSON from the reply with server languages.
  204. void OnServerLangsResponseJsonParsed(
  205. absl::optional<base::Value> json_data,
  206. const absl::optional<std::string>& error);
  207. const std::unique_ptr<Client> client_;
  208. // Maps from request key to previously-obtained annotation results.
  209. // TODO(crbug.com/916420): periodically clear entries from this cache.
  210. std::map<RequestKey, mojom::AnnotateImageResultPtr> cached_results_;
  211. // Maps from request key to its list of request infos (i.e. info of clients
  212. // that have made requests with that language and source ID).
  213. std::map<RequestKey, std::list<ClientRequestInfo>> request_infos_;
  214. // Maps from request keys of images currently being locally processed to the
  215. // ImageProcessors responsible for their processing.
  216. //
  217. // The value is a weak pointer to an entry in the client info list for the
  218. // given request key.
  219. //
  220. // Note that separate local processing will be scheduled for two requests that
  221. // share a source ID but differ in language. This is suboptimal; in future we
  222. // could share local processing among all relevant requests.
  223. std::map<RequestKey, mojo::Remote<mojom::ImageProcessor>*> local_processors_;
  224. // A list of currently-ongoing HTTP requests to the image annotation server.
  225. UrlLoaderList ongoing_server_requests_;
  226. // A queue of requests for the image annotation server waiting to be made.
  227. std::deque<ServerRequestInfo> server_request_queue_;
  228. // The set of request keys for which a server request has been scheduled but
  229. // not yet returned to clients.
  230. //
  231. // This comprises request keys for which:
  232. // - A server query has been queued (but not yet sent),
  233. // - A server query is ongoing,
  234. // - A server query has been returned and is being parsed.
  235. std::set<RequestKey> pending_requests_;
  236. // The request for server languages.
  237. std::unique_ptr<network::SimpleURLLoader> langs_url_loader_;
  238. scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
  239. mojo::ReceiverSet<mojom::Annotator> receivers_;
  240. // Should not be used directly; GetJsonParser() should be called instead.
  241. mojo::Remote<data_decoder::mojom::JsonParser> json_parser_;
  242. // A timer used to throttle server request frequency.
  243. std::unique_ptr<base::RepeatingTimer> server_request_timer_;
  244. const GURL pixels_server_url_;
  245. const GURL langs_server_url_;
  246. const std::string api_key_;
  247. const int batch_size_;
  248. const double min_ocr_confidence_;
  249. // The languages that the server accepts.
  250. std::vector<std::string> server_languages_;
  251. // Used for all callbacks.
  252. base::WeakPtrFactory<Annotator> weak_factory_{this};
  253. };
  254. } // namespace image_annotation
  255. #endif // SERVICES_IMAGE_ANNOTATION_ANNOTATOR_H_