ambient_photo_controller.cc 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/ambient/ambient_photo_controller.h"
  5. #include <algorithm>
  6. #include <array>
  7. #include <string>
  8. #include <utility>
  9. #include <vector>
  10. #include "ash/ambient/ambient_constants.h"
  11. #include "ash/ambient/ambient_controller.h"
  12. #include "ash/ambient/ambient_photo_cache.h"
  13. #include "ash/ambient/ambient_weather_controller.h"
  14. #include "ash/ambient/model/ambient_backend_model.h"
  15. #include "ash/public/cpp/ambient/ambient_backend_controller.h"
  16. #include "ash/public/cpp/ambient/ambient_client.h"
  17. #include "ash/public/cpp/ambient/proto/photo_cache_entry.pb.h"
  18. #include "ash/public/cpp/image_downloader.h"
  19. #include "ash/shell.h"
  20. #include "base/barrier_closure.h"
  21. #include "base/base64.h"
  22. #include "base/base_paths.h"
  23. #include "base/bind.h"
  24. #include "base/callback.h"
  25. #include "base/files/file_path.h"
  26. #include "base/files/file_util.h"
  27. #include "base/guid.h"
  28. #include "base/hash/sha1.h"
  29. #include "base/path_service.h"
  30. #include "base/rand_util.h"
  31. #include "base/strings/string_number_conversions.h"
  32. #include "base/strings/string_util.h"
  33. #include "base/system/sys_info.h"
  34. #include "base/task/task_runner_util.h"
  35. #include "base/task/task_traits.h"
  36. #include "base/task/thread_pool.h"
  37. #include "base/threading/sequenced_task_runner_handle.h"
  38. #include "net/traffic_annotation/network_traffic_annotation.h"
  39. #include "services/network/public/cpp/shared_url_loader_factory.h"
  40. #include "third_party/abseil-cpp/absl/types/optional.h"
  41. #include "ui/gfx/image/image_skia.h"
  42. #include "url/gurl.h"
  43. namespace ash {
  44. namespace {
  45. // TODO(b/161357364): refactor utility functions and constants
  46. constexpr net::BackoffEntry::Policy kResumeFetchImageBackoffPolicy = {
  47. kMaxConsecutiveReadPhotoFailures, // Number of initial errors to ignore.
  48. 500, // Initial delay in ms.
  49. 2.0, // Factor by which the waiting time will be multiplied.
  50. 0.2, // Fuzzing percentage.
  51. 8 * 60 * 1000, // Maximum delay in ms.
  52. -1, // Never discard the entry.
  53. true, // Use initial delay.
  54. };
  55. base::TaskTraits GetTaskTraits() {
  56. return {base::MayBlock(), base::TaskPriority::USER_BLOCKING,
  57. base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN};
  58. }
  59. const std::array<const char*, 2>& GetBackupPhotoUrls() {
  60. return Shell::Get()
  61. ->ambient_controller()
  62. ->ambient_backend_controller()
  63. ->GetBackupPhotoUrls();
  64. }
  65. // Get the cache root path for ambient mode.
  66. base::FilePath GetCacheRootPath() {
  67. base::FilePath home_dir;
  68. CHECK(base::PathService::Get(base::DIR_HOME, &home_dir));
  69. return home_dir.Append(FILE_PATH_LITERAL(kAmbientModeDirectoryName));
  70. }
  71. } // namespace
  72. AmbientPhotoController::AmbientPhotoController(
  73. AmbientClient& ambient_client,
  74. AmbientAccessTokenController& access_token_controller,
  75. AmbientViewDelegate& view_delegate,
  76. AmbientPhotoConfig photo_config)
  77. : ambient_backend_model_(std::move(photo_config)),
  78. resume_fetch_image_backoff_(&kResumeFetchImageBackoffPolicy),
  79. photo_cache_(AmbientPhotoCache::Create(
  80. GetCacheRootPath().Append(
  81. FILE_PATH_LITERAL(kAmbientModeCacheDirectoryName)),
  82. ambient_client,
  83. access_token_controller)),
  84. backup_photo_cache_(AmbientPhotoCache::Create(
  85. GetCacheRootPath().Append(
  86. FILE_PATH_LITERAL(kAmbientModeBackupCacheDirectoryName)),
  87. ambient_client,
  88. access_token_controller)),
  89. task_runner_(
  90. base::ThreadPool::CreateSequencedTaskRunner(GetTaskTraits())) {
  91. scoped_view_delegate_observation_.Observe(&view_delegate);
  92. ScheduleFetchBackupImages();
  93. }
  94. AmbientPhotoController::~AmbientPhotoController() = default;
  95. void AmbientPhotoController::Init(
  96. std::unique_ptr<AmbientTopicQueue::Delegate> topic_queue_delegate) {
  97. state_ = State::kPreparingNextTopicSet;
  98. topic_index_ = 0;
  99. retries_to_read_from_cache_ = kMaxNumberOfCachedImages;
  100. backup_retries_to_read_from_cache_ = GetBackupPhotoUrls().size();
  101. num_topics_prepared_ = 0;
  102. ambient_topic_queue_ = std::make_unique<AmbientTopicQueue>(
  103. /*topic_fetch_limit=*/kMaxNumberOfCachedImages,
  104. /*topic_fetch_size=*/kTopicsBatchSize, kTopicFetchInterval,
  105. ambient_backend_model_.photo_config().should_split_topics,
  106. std::move(topic_queue_delegate),
  107. Shell::Get()->ambient_controller()->ambient_backend_controller());
  108. }
  109. void AmbientPhotoController::StartScreenUpdate(
  110. std::unique_ptr<AmbientTopicQueue::Delegate> topic_queue_delegate) {
  111. if (state_ != State::kInactive) {
  112. DVLOG(3) << "AmbientPhotoController is already active. Ignoring "
  113. "StartScreenUpdate().";
  114. return;
  115. }
  116. Init(std::move(topic_queue_delegate));
  117. FetchWeather();
  118. weather_refresh_timer_.Start(
  119. FROM_HERE, kWeatherRefreshInterval,
  120. base::BindRepeating(&AmbientPhotoController::FetchWeather,
  121. weak_factory_.GetWeakPtr()));
  122. if (backup_photo_refresh_timer_.IsRunning()) {
  123. // Would use |timer_.FireNow()| but this does not execute if screen is
  124. // locked. Manually call the expected callback instead.
  125. backup_photo_refresh_timer_.Stop();
  126. FetchBackupImages();
  127. }
  128. StartPreparingNextTopic();
  129. }
  130. void AmbientPhotoController::StopScreenUpdate() {
  131. state_ = State::kInactive;
  132. weather_refresh_timer_.Stop();
  133. resume_fetch_image_backoff_.Reset();
  134. ambient_backend_model_.Clear();
  135. ambient_topic_queue_.reset();
  136. weak_factory_.InvalidateWeakPtrs();
  137. }
  138. bool AmbientPhotoController::IsScreenUpdateActive() const {
  139. return state_ != State::kInactive;
  140. }
  141. void AmbientPhotoController::OnMarkerHit(AmbientPhotoConfig::Marker marker) {
  142. if (!ambient_backend_model_.photo_config().refresh_topic_markers.contains(
  143. marker)) {
  144. DVLOG(3) << "UI event " << marker
  145. << " does not trigger a topic refresh. Ignoring...";
  146. return;
  147. }
  148. DVLOG(3) << "UI event " << marker << " triggering topic refresh";
  149. if (state_ == State::kInactive) {
  150. LOG(DFATAL) << "Received unexpected UI marker " << marker
  151. << " while inactive";
  152. return;
  153. }
  154. bool is_still_preparing_topics = state_ != State::kWaitingForNextMarker;
  155. state_ = State::kPreparingNextTopicSet;
  156. num_topics_prepared_ = 0;
  157. if (is_still_preparing_topics) {
  158. // The controller is still in the middle of preparing a topic from the
  159. // previous set (i.e. waiting on a callback or timer to fire). Resetting
  160. // |num_topics_prepared_| to 0 above is enough, and the topic currently
  161. // being prepared will count towards the next set.
  162. DVLOG(4) << "Did not finished preparing current topic set in time. "
  163. "Starting new set...";
  164. } else {
  165. StartPreparingNextTopic();
  166. }
  167. }
  168. void AmbientPhotoController::FetchWeather() {
  169. Shell::Get()
  170. ->ambient_controller()
  171. ->ambient_weather_controller()
  172. ->FetchWeather();
  173. }
  174. void AmbientPhotoController::ClearCache() {
  175. DCHECK(photo_cache_);
  176. DCHECK(backup_photo_cache_);
  177. photo_cache_->Clear();
  178. backup_photo_cache_->Clear();
  179. }
  180. void AmbientPhotoController::ScheduleFetchBackupImages() {
  181. DVLOG(3) << __func__;
  182. if (backup_photo_refresh_timer_.IsRunning())
  183. return;
  184. backup_photo_refresh_timer_.Start(
  185. FROM_HERE,
  186. std::max(kBackupPhotoRefreshDelay,
  187. resume_fetch_image_backoff_.GetTimeUntilRelease()),
  188. base::BindOnce(&AmbientPhotoController::FetchBackupImages,
  189. weak_factory_.GetWeakPtr()));
  190. }
  191. void AmbientPhotoController::FetchBackupImages() {
  192. const auto& backup_photo_urls = GetBackupPhotoUrls();
  193. backup_retries_to_read_from_cache_ = backup_photo_urls.size();
  194. for (size_t i = 0; i < backup_photo_urls.size(); i++) {
  195. backup_photo_cache_->DownloadPhotoToFile(
  196. backup_photo_urls.at(i),
  197. /*cache_index=*/i,
  198. base::BindOnce(&AmbientPhotoController::OnBackupImageFetched,
  199. weak_factory_.GetWeakPtr()));
  200. }
  201. }
  202. void AmbientPhotoController::OnBackupImageFetched(bool success) {
  203. if (!success) {
  204. // TODO(b/169807068) Change to retry individual failed images.
  205. resume_fetch_image_backoff_.InformOfRequest(/*succeeded=*/false);
  206. LOG(WARNING) << "Downloading backup image failed.";
  207. ScheduleFetchBackupImages();
  208. return;
  209. }
  210. resume_fetch_image_backoff_.InformOfRequest(/*succeeded=*/true);
  211. }
  212. void AmbientPhotoController::OnTopicsAvailableInQueue(
  213. AmbientTopicQueue::WaitResult wait_result) {
  214. if (state_ != State::kPreparingNextTopicSet)
  215. return;
  216. switch (wait_result) {
  217. case AmbientTopicQueue::WaitResult::kTopicsAvailable:
  218. ReadPhotoFromTopicQueue();
  219. break;
  220. case AmbientTopicQueue::WaitResult::kTopicFetchBackingOff:
  221. case AmbientTopicQueue::WaitResult::kTopicFetchLimitReached:
  222. // If there are no topics in the queue, will try to read from disk cache.
  223. TryReadPhotoFromCache();
  224. break;
  225. }
  226. }
  227. void AmbientPhotoController::ResetImageData() {
  228. cache_entry_.Clear();
  229. image_ = gfx::ImageSkia();
  230. related_image_ = gfx::ImageSkia();
  231. }
  232. void AmbientPhotoController::ReadPhotoFromTopicQueue() {
  233. ResetImageData();
  234. DVLOG(3) << "Downloading topic photos";
  235. AmbientModeTopic topic = ambient_topic_queue_->Pop();
  236. ::ambient::Photo* photo = cache_entry_.mutable_primary_photo();
  237. photo->set_details(topic.details);
  238. photo->set_is_portrait(topic.is_portrait);
  239. photo->set_type(topic.topic_type);
  240. const int num_callbacks = (topic.related_image_url.empty()) ? 1 : 2;
  241. auto on_done = base::BarrierClosure(
  242. num_callbacks,
  243. base::BindOnce(&AmbientPhotoController::OnAllPhotoRawDataDownloaded,
  244. weak_factory_.GetWeakPtr()));
  245. photo_cache_->DownloadPhoto(
  246. topic.url,
  247. base::BindOnce(&AmbientPhotoController::OnPhotoRawDataDownloaded,
  248. weak_factory_.GetWeakPtr(),
  249. /*is_related_image=*/false, on_done));
  250. if (!topic.related_image_url.empty()) {
  251. ::ambient::Photo* photo = cache_entry_.mutable_related_photo();
  252. photo->set_details(topic.related_details);
  253. photo->set_is_portrait(topic.is_portrait);
  254. photo->set_type(topic.topic_type);
  255. photo_cache_->DownloadPhoto(
  256. topic.related_image_url,
  257. base::BindOnce(&AmbientPhotoController::OnPhotoRawDataDownloaded,
  258. weak_factory_.GetWeakPtr(),
  259. /*is_related_image=*/true, on_done));
  260. }
  261. }
  262. void AmbientPhotoController::TryReadPhotoFromCache() {
  263. ResetImageData();
  264. // Stop reading from cache after the max number of retries.
  265. if (retries_to_read_from_cache_ == 0) {
  266. if (backup_retries_to_read_from_cache_ == 0) {
  267. LOG(WARNING) << "Failed to read from cache";
  268. ambient_backend_model_.AddImageFailure();
  269. // Do not refresh image if image loading has failed repeatedly, or there
  270. // are no more topics to retry. Note |ambient_topic_queue_| may be null
  271. // if AddImageFailure() ultimately led to an AmbientBackendModelObserver
  272. // calling StopScreenUpdate().
  273. if (ambient_backend_model_.ImageLoadingFailed() ||
  274. !ambient_topic_queue_ || ambient_topic_queue_->IsEmpty()) {
  275. LOG(WARNING) << "Not attempting image refresh";
  276. return;
  277. }
  278. // Try to resume normal workflow with backoff.
  279. const base::TimeDelta delay =
  280. resume_fetch_image_backoff_.GetTimeUntilRelease();
  281. base::SequencedTaskRunnerHandle::Get()->PostDelayedTask(
  282. FROM_HERE,
  283. base::BindOnce(&AmbientPhotoController::StartPreparingNextTopic,
  284. weak_factory_.GetWeakPtr()),
  285. delay);
  286. return;
  287. }
  288. --backup_retries_to_read_from_cache_;
  289. DVLOG(3) << "Read from backup cache index: "
  290. << backup_cache_index_for_display_;
  291. // Try to read a backup image.
  292. backup_photo_cache_->ReadPhotoCache(
  293. /*cache_index=*/backup_cache_index_for_display_,
  294. base::BindOnce(&AmbientPhotoController::OnPhotoCacheReadComplete,
  295. weak_factory_.GetWeakPtr()));
  296. backup_cache_index_for_display_++;
  297. if (backup_cache_index_for_display_ == GetBackupPhotoUrls().size())
  298. backup_cache_index_for_display_ = 0;
  299. return;
  300. }
  301. --retries_to_read_from_cache_;
  302. int current_cache_index = cache_index_for_display_;
  303. ++cache_index_for_display_;
  304. if (cache_index_for_display_ == kMaxNumberOfCachedImages)
  305. cache_index_for_display_ = 0;
  306. DVLOG(3) << "Read from cache index: " << current_cache_index;
  307. photo_cache_->ReadPhotoCache(
  308. current_cache_index,
  309. base::BindOnce(&AmbientPhotoController::OnPhotoCacheReadComplete,
  310. weak_factory_.GetWeakPtr()));
  311. }
  312. void AmbientPhotoController::OnPhotoCacheReadComplete(
  313. ::ambient::PhotoCacheEntry cache_entry) {
  314. cache_entry_ = std::move(cache_entry);
  315. OnAllPhotoRawDataAvailable(/*from_downloading=*/false);
  316. }
  317. void AmbientPhotoController::OnPhotoRawDataDownloaded(
  318. bool is_related_image,
  319. base::RepeatingClosure on_done,
  320. std::string&& data) {
  321. if (is_related_image)
  322. cache_entry_.mutable_related_photo()->set_image(std::move(data));
  323. else
  324. cache_entry_.mutable_primary_photo()->set_image(std::move(data));
  325. std::move(on_done).Run();
  326. }
  327. void AmbientPhotoController::OnAllPhotoRawDataDownloaded() {
  328. DVLOG(3) << __func__;
  329. OnAllPhotoRawDataAvailable(/*from_downloading=*/true);
  330. }
  331. void AmbientPhotoController::OnAllPhotoRawDataAvailable(bool from_downloading) {
  332. if (!cache_entry_.has_primary_photo() ||
  333. cache_entry_.primary_photo().image().empty()) {
  334. if (from_downloading) {
  335. LOG(ERROR) << "Failed to download image";
  336. resume_fetch_image_backoff_.InformOfRequest(/*succeeded=*/false);
  337. }
  338. // Try to read from cache when failure happens.
  339. TryReadPhotoFromCache();
  340. return;
  341. }
  342. if (from_downloading) {
  343. // If the data is fetched from downloading, write to disk.
  344. // Note: WritePhotoCache could fail. The saved file name may not be
  345. // continuous.
  346. DVLOG(3) << "Save photo to cache index: " << cache_index_for_store_;
  347. auto current_cache_index = cache_index_for_store_;
  348. ++cache_index_for_store_;
  349. if (cache_index_for_store_ == kMaxNumberOfCachedImages)
  350. cache_index_for_store_ = 0;
  351. photo_cache_->WritePhotoCache(
  352. /*cache_index=*/current_cache_index, cache_entry_,
  353. base::BindOnce(&AmbientPhotoController::OnPhotoRawDataSaved,
  354. weak_factory_.GetWeakPtr(), from_downloading));
  355. } else {
  356. OnPhotoRawDataSaved(from_downloading);
  357. }
  358. }
  359. void AmbientPhotoController::OnPhotoRawDataSaved(bool from_downloading) {
  360. const bool has_related = cache_entry_.has_related_photo() &&
  361. !cache_entry_.related_photo().image().empty();
  362. const int num_callbacks = has_related ? 2 : 1;
  363. auto on_done = base::BarrierClosure(
  364. num_callbacks,
  365. base::BindOnce(
  366. &AmbientPhotoController::OnAllPhotoDecoded,
  367. weak_factory_.GetWeakPtr(), from_downloading,
  368. /*hash=*/base::SHA1HashString(cache_entry_.primary_photo().image())));
  369. DecodePhotoRawData(from_downloading,
  370. /*is_related_image=*/false, on_done,
  371. cache_entry_.primary_photo().image());
  372. if (has_related) {
  373. DecodePhotoRawData(from_downloading, /*is_related_image=*/true, on_done,
  374. cache_entry_.related_photo().image());
  375. }
  376. }
  377. void AmbientPhotoController::DecodePhotoRawData(bool from_downloading,
  378. bool is_related_image,
  379. base::RepeatingClosure on_done,
  380. const std::string& data) {
  381. photo_cache_->DecodePhoto(
  382. data, base::BindOnce(&AmbientPhotoController::OnPhotoDecoded,
  383. weak_factory_.GetWeakPtr(), from_downloading,
  384. is_related_image, std::move(on_done)));
  385. }
  386. void AmbientPhotoController::OnPhotoDecoded(bool from_downloading,
  387. bool is_related_image,
  388. base::RepeatingClosure on_done,
  389. const gfx::ImageSkia& image) {
  390. if (is_related_image)
  391. related_image_ = image;
  392. else
  393. image_ = image;
  394. std::move(on_done).Run();
  395. }
  396. void AmbientPhotoController::OnAllPhotoDecoded(bool from_downloading,
  397. const std::string& hash) {
  398. DVLOG(3) << __func__;
  399. if (image_.isNull()) {
  400. LOG(WARNING) << "Image decoding failed";
  401. if (from_downloading)
  402. resume_fetch_image_backoff_.InformOfRequest(/*succeeded=*/false);
  403. // Try to read from cache when failure happens.
  404. TryReadPhotoFromCache();
  405. return;
  406. } else if (ambient_backend_model_.IsHashDuplicate(hash)) {
  407. LOG(WARNING) << "Skipping loading duplicate image.";
  408. TryReadPhotoFromCache();
  409. return;
  410. }
  411. retries_to_read_from_cache_ = kMaxNumberOfCachedImages;
  412. backup_retries_to_read_from_cache_ = GetBackupPhotoUrls().size();
  413. if (from_downloading)
  414. resume_fetch_image_backoff_.InformOfRequest(/*succeeded=*/true);
  415. PhotoWithDetails detailed_photo;
  416. detailed_photo.photo = image_;
  417. detailed_photo.related_photo = related_image_;
  418. detailed_photo.details = cache_entry_.primary_photo().details();
  419. detailed_photo.related_details = cache_entry_.related_photo().details();
  420. detailed_photo.is_portrait = cache_entry_.primary_photo().is_portrait();
  421. detailed_photo.topic_type = cache_entry_.primary_photo().type();
  422. detailed_photo.hash = hash;
  423. ResetImageData();
  424. if (state_ != State::kPreparingNextTopicSet) {
  425. LOG(ERROR) << "Topic prepared when controller should be idle in state "
  426. << state_;
  427. return;
  428. }
  429. // AddNextImage() can call out to observers, who can synchronously interact
  430. // with the controller within their observer notification methods. So the
  431. // internal |state_| should be updated before calling AddNextImage() so that
  432. // it is consistent with the model.
  433. size_t target_num_topics_to_prepare =
  434. ambient_backend_model_.ImagesReady()
  435. ? ambient_backend_model_.photo_config().topic_set_size
  436. : ambient_backend_model_.photo_config().GetNumDecodedTopicsToBuffer();
  437. ++num_topics_prepared_;
  438. if (num_topics_prepared_ >= target_num_topics_to_prepare)
  439. state_ = State::kWaitingForNextMarker;
  440. ambient_backend_model_.AddNextImage(std::move(detailed_photo));
  441. if (state_ == State::kPreparingNextTopicSet)
  442. StartPreparingNextTopic();
  443. }
  444. void AmbientPhotoController::FetchTopicsForTesting() {
  445. StartPreparingNextTopic();
  446. }
  447. void AmbientPhotoController::FetchImageForTesting() {
  448. if (!ambient_topic_queue_->IsEmpty()) {
  449. ReadPhotoFromTopicQueue();
  450. } else {
  451. TryReadPhotoFromCache();
  452. }
  453. }
  454. void AmbientPhotoController::FetchBackupImagesForTesting() {
  455. FetchBackupImages();
  456. }
  457. void AmbientPhotoController::StartPreparingNextTopic() {
  458. DCHECK_EQ(state_, State::kPreparingNextTopicSet);
  459. ambient_topic_queue_->WaitForTopicsAvailable(
  460. base::BindOnce(&AmbientPhotoController::OnTopicsAvailableInQueue,
  461. weak_factory_.GetWeakPtr()));
  462. }
  463. std::ostream& operator<<(std::ostream& os,
  464. AmbientPhotoController::State state) {
  465. switch (state) {
  466. case AmbientPhotoController::State::kInactive:
  467. return os << "INACTIVE";
  468. case AmbientPhotoController::State::kWaitingForNextMarker:
  469. return os << "WAITING_FOR_NEXT_MARKER";
  470. case AmbientPhotoController::State::kPreparingNextTopicSet:
  471. return os << "PREPARING_NEXT_TOPIC_SET";
  472. }
  473. }
  474. } // namespace ash