projector_controller_unittest.cc 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/projector/projector_controller_impl.h"
  5. #include <initializer_list>
  6. #include <memory>
  7. #include <string>
  8. #include <vector>
  9. #include "ash/constants/ash_features.h"
  10. #include "ash/projector/model/projector_session_impl.h"
  11. #include "ash/projector/projector_metadata_controller.h"
  12. #include "ash/projector/projector_metrics.h"
  13. #include "ash/projector/test/mock_projector_metadata_controller.h"
  14. #include "ash/projector/test/mock_projector_ui_controller.h"
  15. #include "ash/public/cpp/projector/projector_new_screencast_precondition.h"
  16. #include "ash/public/cpp/projector/projector_session.h"
  17. #include "ash/public/cpp/test/mock_projector_client.h"
  18. #include "ash/shell.h"
  19. #include "ash/test/ash_test_base.h"
  20. #include "base/bind.h"
  21. #include "base/callback_forward.h"
  22. #include "base/files/file.h"
  23. #include "base/files/file_path.h"
  24. #include "base/files/file_util.h"
  25. #include "base/files/scoped_temp_dir.h"
  26. #include "base/json/json_writer.h"
  27. #include "base/run_loop.h"
  28. #include "base/test/bind.h"
  29. #include "base/test/metrics/histogram_tester.h"
  30. #include "base/test/scoped_feature_list.h"
  31. #include "base/time/time.h"
  32. #include "base/values.h"
  33. #include "chromeos/ash/components/audio/cras_audio_handler.h"
  34. #include "chromeos/ash/components/dbus/audio/audio_node.h"
  35. #include "chromeos/ash/components/dbus/audio/fake_cras_audio_client.h"
  36. #include "media/mojo/mojom/speech_recognition_result.h"
  37. #include "media/mojo/mojom/speech_recognition_service.mojom.h"
  38. #include "testing/gtest/include/gtest/gtest.h"
  39. #include "third_party/skia/include/core/SkColor.h"
  40. #include "ui/gfx/image/image_unittest_util.h"
  41. namespace ash {
  42. namespace {
  43. using testing::_;
  44. using testing::ElementsAre;
  45. struct AudioNodeInfo {
  46. bool is_input;
  47. uint64_t id;
  48. const char* const device_name;
  49. const char* const type;
  50. const char* const name;
  51. };
  52. constexpr char kProjectorCreationFlowHistogramName[] =
  53. "Ash.Projector.CreationFlow.ClamshellMode";
  54. constexpr char kProjectorTranscriptsCountHistogramName[] =
  55. "Ash.Projector.TranscriptsCount.ClamshellMode";
  56. constexpr char kMetadataFileName[] = "MyScreencast";
  57. constexpr char kProjectorExtension[] = "projector";
  58. void NotifyControllerForFinalSpeechResult(ProjectorControllerImpl* controller) {
  59. media::SpeechRecognitionResult result;
  60. result.transcription = "transcript text 1";
  61. result.is_final = true;
  62. result.timing_information = media::TimingInformation();
  63. result.timing_information->audio_start_time = base::Milliseconds(0);
  64. result.timing_information->audio_end_time = base::Milliseconds(3000);
  65. std::vector<media::HypothesisParts> hypothesis_parts;
  66. std::string hypothesis_text[3] = {"transcript", "text", "1"};
  67. int hypothesis_time[3] = {1000, 2000, 2500};
  68. for (int i = 0; i < 3; i++) {
  69. hypothesis_parts.emplace_back(
  70. std::vector<std::string>({hypothesis_text[i]}),
  71. base::Milliseconds(hypothesis_time[i]));
  72. }
  73. result.timing_information->hypothesis_parts = std::move(hypothesis_parts);
  74. controller->OnTranscription(result);
  75. }
  76. void NotifyControllerForPartialSpeechResult(
  77. ProjectorControllerImpl* controller) {
  78. controller->OnTranscription(
  79. media::SpeechRecognitionResult("transcript partial text 1", false));
  80. }
  81. class ProjectorMetadataControllerForTest : public ProjectorMetadataController {
  82. public:
  83. ProjectorMetadataControllerForTest() = default;
  84. ProjectorMetadataControllerForTest(
  85. const ProjectorMetadataControllerForTest&) = delete;
  86. ProjectorMetadataControllerForTest& operator=(
  87. const ProjectorMetadataControllerForTest&) = delete;
  88. ~ProjectorMetadataControllerForTest() override = default;
  89. void SetRunLoopQuitClosure(base::RepeatingClosure closure) {
  90. quit_closure_ = base::BindOnce(closure);
  91. }
  92. protected:
  93. // ProjectorMetadataController:
  94. void OnSaveFileResult(const base::FilePath& path,
  95. size_t transcripts_count,
  96. bool success) override {
  97. ProjectorMetadataController::OnSaveFileResult(path, transcripts_count,
  98. success);
  99. std::move(quit_closure_).Run();
  100. }
  101. private:
  102. base::OnceClosure quit_closure_;
  103. };
  104. } // namespace
  105. class ProjectorControllerTest : public AshTestBase {
  106. public:
  107. ProjectorControllerTest()
  108. : AshTestBase(base::test::TaskEnvironment::TimeSource::MOCK_TIME) {
  109. scoped_feature_list_.InitWithFeatures(
  110. {features::kProjector, features::kProjectorAnnotator}, {});
  111. }
  112. ProjectorControllerTest(const ProjectorControllerTest&) = delete;
  113. ProjectorControllerTest& operator=(const ProjectorControllerTest&) = delete;
  114. // AshTestBase:
  115. void SetUp() override {
  116. AshTestBase::SetUp();
  117. controller_ =
  118. static_cast<ProjectorControllerImpl*>(ProjectorController::Get());
  119. auto mock_ui_controller =
  120. std::make_unique<MockProjectorUiController>(controller_);
  121. mock_ui_controller_ = mock_ui_controller.get();
  122. controller_->SetProjectorUiControllerForTest(std::move(mock_ui_controller));
  123. auto mock_metadata_controller =
  124. std::make_unique<MockProjectorMetadataController>();
  125. mock_metadata_controller_ = mock_metadata_controller.get();
  126. controller_->SetProjectorMetadataControllerForTest(
  127. std::move(mock_metadata_controller));
  128. controller_->SetClient(&mock_client_);
  129. controller_->OnSpeechRecognitionAvailabilityChanged(
  130. SpeechRecognitionAvailability::kAvailable);
  131. }
  132. void InitializeRealMetadataController() {
  133. std::unique_ptr<ProjectorMetadataController> metadata_controller =
  134. std::make_unique<ProjectorMetadataControllerForTest>();
  135. metadata_controller_ = static_cast<ProjectorMetadataControllerForTest*>(
  136. metadata_controller.get());
  137. controller_->SetProjectorMetadataControllerForTest(
  138. std::move(metadata_controller));
  139. }
  140. protected:
  141. MockProjectorUiController* mock_ui_controller_ = nullptr;
  142. MockProjectorMetadataController* mock_metadata_controller_ = nullptr;
  143. ProjectorMetadataControllerForTest* metadata_controller_;
  144. ProjectorControllerImpl* controller_;
  145. MockProjectorClient mock_client_;
  146. base::HistogramTester histogram_tester_;
  147. base::ScopedTempDir temp_dir_;
  148. private:
  149. base::test::ScopedFeatureList scoped_feature_list_;
  150. };
  151. TEST_F(ProjectorControllerTest, OnTranscription) {
  152. // Verify that |RecordTranscription| in |ProjectorMetadataController| is
  153. // called to record the transcript.
  154. EXPECT_CALL(*mock_metadata_controller_, RecordTranscription(_)).Times(1);
  155. NotifyControllerForFinalSpeechResult(controller_);
  156. }
  157. TEST_F(ProjectorControllerTest, OnTranscriptionPartialResult) {
  158. // Verify that |RecordTranscription| in |ProjectorMetadataController| is not
  159. // called since it is not a final result.
  160. EXPECT_CALL(*mock_metadata_controller_, RecordTranscription(_)).Times(0);
  161. NotifyControllerForPartialSpeechResult(controller_);
  162. }
  163. TEST_F(ProjectorControllerTest, OnAudioNodesChanged) {
  164. ON_CALL(mock_client_, IsDriveFsMounted())
  165. .WillByDefault(testing::Return(true));
  166. const AudioNodeInfo kInternalMic[] = {
  167. {true, 55555, "Fake Mic", "INTERNAL_MIC", "Internal Mic"}};
  168. const AudioNode audio_node = AudioNode(
  169. kInternalMic->is_input, kInternalMic->id,
  170. /*has_v2_stable_device_id=*/false, kInternalMic->id,
  171. /*stable_device_id_v2=*/0, kInternalMic->device_name, kInternalMic->type,
  172. kInternalMic->name, /*active=*/false,
  173. /*plugged_time=*/0, /*max_supported_channels=*/1, /*audio_effect=*/1);
  174. FakeCrasAudioClient::Get()->SetAudioNodesForTesting({audio_node});
  175. CrasAudioHandler::Get()->SetActiveInputNodes({kInternalMic->id});
  176. EXPECT_CALL(mock_client_,
  177. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  178. NewScreencastPreconditionState::kEnabled, {})));
  179. controller_->OnAudioNodesChanged();
  180. CrasAudioHandler::Get()->SetActiveInputNodes({});
  181. EXPECT_CALL(mock_client_,
  182. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  183. NewScreencastPreconditionState::kDisabled,
  184. {NewScreencastPreconditionReason::kNoMic})));
  185. controller_->OnAudioNodesChanged();
  186. }
  187. TEST_F(ProjectorControllerTest, OnSpeechRecognitionAvailabilityChanged) {
  188. controller_->OnSpeechRecognitionAvailabilityChanged(
  189. SpeechRecognitionAvailability::kAvailable);
  190. EXPECT_TRUE(controller_->IsEligible());
  191. controller_->OnSpeechRecognitionAvailabilityChanged(
  192. SpeechRecognitionAvailability::kOnDeviceSpeechRecognitionNotSupported);
  193. EXPECT_FALSE(controller_->IsEligible());
  194. }
  195. TEST_F(ProjectorControllerTest, EnableAnnotatorTool) {
  196. // Verify that |OnMarkerPressed| in |ProjectorUiController| is called.
  197. EXPECT_CALL(*mock_ui_controller_, EnableAnnotatorTool());
  198. controller_->EnableAnnotatorTool();
  199. }
  200. TEST_F(ProjectorControllerTest, SetAnnotatorTool) {
  201. AnnotatorTool tool;
  202. // Verify that |SetAnnotatorTool| in |ProjectorUiController| is called.
  203. EXPECT_CALL(*mock_ui_controller_, SetAnnotatorTool(tool));
  204. controller_->SetAnnotatorTool(tool);
  205. }
  206. TEST_F(ProjectorControllerTest, RecordingStarted) {
  207. EXPECT_CALL(mock_client_, StartSpeechRecognition());
  208. EXPECT_CALL(*mock_metadata_controller_, OnRecordingStarted());
  209. // Verify that |ShowAnnotationTray| in |ProjectorUiController| is called.
  210. auto* root = Shell::GetPrimaryRootWindow();
  211. EXPECT_CALL(*mock_ui_controller_, ShowAnnotationTray(root)).Times(1);
  212. controller_->OnRecordingStarted(root, /*is_in_projector_mode=*/true);
  213. histogram_tester_.ExpectUniqueSample(
  214. kProjectorCreationFlowHistogramName,
  215. /*sample=*/ProjectorCreationFlow::kRecordingStarted, /*count=*/1);
  216. }
  217. TEST_F(ProjectorControllerTest, RecordingEnded) {
  218. base::FilePath screencast_container_path;
  219. ASSERT_TRUE(mock_client_.GetBaseStoragePath(&screencast_container_path));
  220. ON_CALL(mock_client_, IsDriveFsMounted())
  221. .WillByDefault(testing::Return(true));
  222. // Verify that |HideAnnotationTray| in |ProjectorUiController| is
  223. // called.
  224. EXPECT_CALL(*mock_ui_controller_, HideAnnotationTray()).Times(1);
  225. EXPECT_CALL(mock_client_, OpenProjectorApp()).Times(0);
  226. EXPECT_CALL(mock_client_,
  227. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  228. NewScreencastPreconditionState::kDisabled,
  229. {NewScreencastPreconditionReason::kInProjectorSession})));
  230. controller_->projector_session()->Start("projector_data");
  231. histogram_tester_.ExpectUniqueSample(
  232. kProjectorCreationFlowHistogramName,
  233. /*sample=*/ProjectorCreationFlow::kSessionStarted, /*count=*/1);
  234. controller_->OnRecordingStarted(Shell::GetPrimaryRootWindow(),
  235. /*is_in_projector_mode=*/true);
  236. histogram_tester_.ExpectBucketCount(
  237. kProjectorCreationFlowHistogramName,
  238. /*sample=*/ProjectorCreationFlow::kRecordingStarted, /*count=*/1);
  239. base::RunLoop runLoop;
  240. controller_->CreateScreencastContainerFolder(base::BindLambdaForTesting(
  241. [&](const base::FilePath& screencast_file_path_no_extension) {
  242. EXPECT_CALL(
  243. mock_client_,
  244. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  245. NewScreencastPreconditionState::kEnabled, {})))
  246. .Times(0);
  247. EXPECT_CALL(mock_client_, StopSpeechRecognition())
  248. .WillOnce(testing::Invoke(
  249. [&]() { controller_->OnSpeechRecognitionStopped(); }));
  250. EXPECT_CALL(*mock_metadata_controller_, SaveMetadata(_)).Times(0);
  251. controller_->OnRecordingEnded(/*is_in_projector_mode=*/true);
  252. runLoop.Quit();
  253. }));
  254. runLoop.Run();
  255. histogram_tester_.ExpectBucketCount(
  256. kProjectorCreationFlowHistogramName,
  257. /*sample=*/ProjectorCreationFlow::kRecordingEnded, /*count=*/1);
  258. histogram_tester_.ExpectTotalCount(kProjectorCreationFlowHistogramName,
  259. /*count=*/3);
  260. }
  261. class ProjectorOnDlpRestrictionCheckedAtVideoEndTest
  262. : public ::testing::WithParamInterface<::testing::tuple<bool, bool>>,
  263. public ProjectorControllerTest {
  264. public:
  265. ProjectorOnDlpRestrictionCheckedAtVideoEndTest() = default;
  266. ProjectorOnDlpRestrictionCheckedAtVideoEndTest(
  267. const ProjectorOnDlpRestrictionCheckedAtVideoEndTest&) = delete;
  268. ProjectorOnDlpRestrictionCheckedAtVideoEndTest& operator=(
  269. const ProjectorOnDlpRestrictionCheckedAtVideoEndTest&) = delete;
  270. ~ProjectorOnDlpRestrictionCheckedAtVideoEndTest() override = default;
  271. };
  272. TEST_P(ProjectorOnDlpRestrictionCheckedAtVideoEndTest, WrapUpRecordingOnce) {
  273. bool wrap_up_by_speech_stopped = std::get<0>(GetParam());
  274. bool user_deleted_video_file = std::get<1>(GetParam());
  275. base::FilePath screencast_container_path;
  276. ASSERT_TRUE(mock_client_.GetBaseStoragePath(&screencast_container_path));
  277. ON_CALL(mock_client_, IsDriveFsMounted())
  278. .WillByDefault(testing::Return(true));
  279. EXPECT_CALL(mock_client_, OpenProjectorApp());
  280. EXPECT_CALL(mock_client_,
  281. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  282. NewScreencastPreconditionState::kDisabled,
  283. {NewScreencastPreconditionReason::kInProjectorSession})));
  284. // Advance clock to 20:02:10 Jan 2nd, 2021.
  285. base::Time start_time;
  286. EXPECT_TRUE(base::Time::FromString("2 Jan 2021 20:02:10", &start_time));
  287. base::TimeDelta forward_by = start_time - base::Time::Now();
  288. task_environment()->AdvanceClock(forward_by);
  289. controller_->projector_session()->Start("projector_data");
  290. histogram_tester_.ExpectUniqueSample(
  291. kProjectorCreationFlowHistogramName,
  292. /*sample=*/ProjectorCreationFlow::kSessionStarted, /*count=*/1);
  293. controller_->OnRecordingStarted(Shell::GetPrimaryRootWindow(),
  294. /*is_in_projector_mode=*/true);
  295. histogram_tester_.ExpectBucketCount(
  296. kProjectorCreationFlowHistogramName,
  297. /*sample=*/ProjectorCreationFlow::kRecordingStarted, /*count=*/1);
  298. base::RunLoop runLoop;
  299. controller_->CreateScreencastContainerFolder(base::BindLambdaForTesting(
  300. [&](const base::FilePath& screencast_file_path_no_extension) {
  301. EXPECT_CALL(
  302. mock_client_,
  303. OnNewScreencastPreconditionChanged(NewScreencastPrecondition(
  304. NewScreencastPreconditionState::kEnabled, {})));
  305. if (!user_deleted_video_file) {
  306. // Verify that |SaveMetadata| in |ProjectorMetadataController| is
  307. // called with the expected path.
  308. const std::string expected_screencast_name =
  309. "Screencast 2021-01-02 20.02.10";
  310. const base::FilePath expected_path =
  311. screencast_container_path.Append("root")
  312. .Append("projector_data")
  313. // Screencast container folder.
  314. .Append(expected_screencast_name)
  315. // Screencast file name without extension.
  316. .Append(expected_screencast_name);
  317. EXPECT_EQ(screencast_file_path_no_extension, expected_path);
  318. // Verify that save metadata only triggered once.
  319. EXPECT_CALL(*mock_metadata_controller_, SaveMetadata(expected_path))
  320. .Times(1);
  321. // Verify that thumbnail file is saved.
  322. controller_->SetOnFileSavedCallbackForTest(base::BindLambdaForTesting(
  323. [&](const base::FilePath& path, bool success) {
  324. EXPECT_TRUE(success);
  325. EXPECT_TRUE(base::PathExists(path));
  326. }));
  327. } else {
  328. // Verify that save metadata is not triggered.
  329. EXPECT_CALL(*mock_metadata_controller_, SaveMetadata(_)).Times(0);
  330. // Verify that Projector Folder is cleaned up.
  331. controller_->SetOnPathDeletedCallbackForTest(
  332. base::BindLambdaForTesting(
  333. [&](const base::FilePath& path, bool success) {
  334. EXPECT_TRUE(success);
  335. EXPECT_FALSE(base::PathExists(path));
  336. }));
  337. }
  338. auto image = gfx::test::CreateImageSkia(10, 10);
  339. if (wrap_up_by_speech_stopped) {
  340. controller_->OnDlpRestrictionCheckedAtVideoEnd(
  341. /*is_in_projector_mode=*/true,
  342. /*user_deleted_video_file=*/user_deleted_video_file,
  343. /*thumbnail=*/image);
  344. controller_->OnSpeechRecognitionStopped();
  345. } else {
  346. controller_->OnSpeechRecognitionStopped();
  347. controller_->OnDlpRestrictionCheckedAtVideoEnd(
  348. /*is_in_projector_mode=*/true,
  349. /*user_deleted_video_file=*/user_deleted_video_file,
  350. /*thumbnail=*/image);
  351. }
  352. runLoop.Quit();
  353. }));
  354. runLoop.Run();
  355. histogram_tester_.ExpectTotalCount(kProjectorCreationFlowHistogramName,
  356. /*count=*/3);
  357. }
  358. INSTANTIATE_TEST_SUITE_P(WrapUpRecordingOnce,
  359. ProjectorOnDlpRestrictionCheckedAtVideoEndTest,
  360. ::testing::Combine(::testing::Bool(),
  361. ::testing::Bool()));
  362. TEST_F(ProjectorControllerTest, NoTranscriptsTest) {
  363. InitializeRealMetadataController();
  364. metadata_controller_->OnRecordingStarted();
  365. base::RunLoop run_loop;
  366. metadata_controller_->SetRunLoopQuitClosure(run_loop.QuitClosure());
  367. // Simulate ending the recording and saving the metadata file.
  368. ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
  369. base::FilePath metadata_file(temp_dir_.GetPath().Append(kMetadataFileName));
  370. metadata_controller_->SaveMetadata(metadata_file);
  371. run_loop.Run();
  372. histogram_tester_.ExpectUniqueSample(kProjectorTranscriptsCountHistogramName,
  373. /*sample=*/0, /*count=*/1);
  374. // Verify the written metadata file size is between 0-100 bytes. Change this
  375. // limit as needed if you make significant changes to the metadata file.
  376. base::File file(metadata_file.AddExtension(kProjectorExtension),
  377. base::File::FLAG_OPEN | base::File::FLAG_READ);
  378. EXPECT_GT(file.GetLength(), 0);
  379. EXPECT_LT(file.GetLength(), 100);
  380. }
  381. TEST_F(ProjectorControllerTest, TranscriptsTest) {
  382. InitializeRealMetadataController();
  383. metadata_controller_->OnRecordingStarted();
  384. base::RunLoop run_loop;
  385. metadata_controller_->SetRunLoopQuitClosure(run_loop.QuitClosure());
  386. // Simulate adding some transcripts.
  387. NotifyControllerForFinalSpeechResult(controller_);
  388. NotifyControllerForFinalSpeechResult(controller_);
  389. // Simulate ending the recording and saving the metadata file.
  390. ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
  391. base::FilePath metadata_file(temp_dir_.GetPath().Append(kMetadataFileName));
  392. metadata_controller_->SaveMetadata(metadata_file);
  393. run_loop.Run();
  394. histogram_tester_.ExpectUniqueSample(kProjectorTranscriptsCountHistogramName,
  395. /*sample=*/2, /*count=*/1);
  396. // Verify the written metadata file size is between 400-500 bytes. This file
  397. // should be larger than the one in the NoTranscriptsTest above. Change this
  398. // limit as needed if you make significant changes to the metadata file.
  399. base::File file(metadata_file.AddExtension(kProjectorExtension),
  400. base::File::FLAG_OPEN | base::File::FLAG_READ);
  401. EXPECT_GT(file.GetLength(), 400);
  402. EXPECT_LT(file.GetLength(), 500);
  403. }
  404. TEST_F(ProjectorControllerTest, OnDriveMountFailed) {
  405. ON_CALL(mock_client_, IsDriveFsMountFailed())
  406. .WillByDefault(testing::Return(true));
  407. ON_CALL(mock_client_, IsDriveFsMounted())
  408. .WillByDefault(testing::Return(false));
  409. EXPECT_EQ(NewScreencastPrecondition(
  410. NewScreencastPreconditionState::kDisabled,
  411. {NewScreencastPreconditionReason::kDriveFsMountFailed}),
  412. controller_->GetNewScreencastPrecondition());
  413. }
  414. } // namespace ash