finder_test.go 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package finder
  15. import (
  16. "fmt"
  17. "io/ioutil"
  18. "log"
  19. "os"
  20. "path/filepath"
  21. "sort"
  22. "strings"
  23. "testing"
  24. "android/soong/finder/fs"
  25. )
  26. // some utils for tests to use
  27. func newFs() *fs.MockFs {
  28. return fs.NewMockFs(map[string][]byte{})
  29. }
  30. func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder {
  31. return newFinderWithNumThreads(t, filesystem, cacheParams, 2)
  32. }
  33. func newFinderWithNumThreads(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) *Finder {
  34. f, err := newFinderAndErr(t, filesystem, cacheParams, numThreads)
  35. if err != nil {
  36. t.Fatal(err.Error())
  37. }
  38. return f
  39. }
  40. func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) (*Finder, error) {
  41. cachePath := "/finder/finder-db"
  42. cacheDir := filepath.Dir(cachePath)
  43. filesystem.MkDirs(cacheDir)
  44. if cacheParams.WorkingDirectory == "" {
  45. cacheParams.WorkingDirectory = "/cwd"
  46. }
  47. logger := log.New(ioutil.Discard, "", 0)
  48. f, err := newImpl(cacheParams, filesystem, logger, cachePath, numThreads)
  49. return f, err
  50. }
  51. func finderWithSameParams(t *testing.T, original *Finder) *Finder {
  52. f, err := finderAndErrorWithSameParams(t, original)
  53. if err != nil {
  54. t.Fatal(err.Error())
  55. }
  56. return f
  57. }
  58. func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) {
  59. f, err := newImpl(
  60. original.cacheMetadata.Config.CacheParams,
  61. original.filesystem,
  62. original.logger,
  63. original.DbPath,
  64. original.numDbLoadingThreads,
  65. )
  66. return f, err
  67. }
  68. // runSimpleTests creates a few files, searches for findme.txt, and checks for the expected matches
  69. func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []string) {
  70. filesystem := newFs()
  71. root := "/tmp"
  72. filesystem.MkDirs(root)
  73. for _, path := range existentPaths {
  74. fs.Create(t, filepath.Join(root, path), filesystem)
  75. }
  76. finder := newFinder(t,
  77. filesystem,
  78. CacheParams{
  79. "/cwd",
  80. []string{root},
  81. false,
  82. nil,
  83. nil,
  84. []string{"findme.txt", "skipme.txt"},
  85. nil,
  86. },
  87. )
  88. defer finder.Shutdown()
  89. foundPaths := finder.FindNamedAt(root, "findme.txt")
  90. absoluteMatches := []string{}
  91. for i := range expectedMatches {
  92. absoluteMatches = append(absoluteMatches, filepath.Join(root, expectedMatches[i]))
  93. }
  94. fs.AssertSameResponse(t, foundPaths, absoluteMatches)
  95. }
  96. // runTestWithSuffixes creates a few files, searches for findme.txt or any file
  97. // with suffix `.findme_ext` and checks for the expected matches
  98. func runTestWithSuffixes(t *testing.T, existentPaths []string, expectedMatches []string) {
  99. filesystem := newFs()
  100. root := "/tmp"
  101. filesystem.MkDirs(root)
  102. for _, path := range existentPaths {
  103. fs.Create(t, filepath.Join(root, path), filesystem)
  104. }
  105. finder := newFinder(t,
  106. filesystem,
  107. CacheParams{
  108. "/cwd",
  109. []string{root},
  110. false,
  111. nil,
  112. nil,
  113. []string{"findme.txt", "skipme.txt"},
  114. []string{".findme_ext"},
  115. },
  116. )
  117. defer finder.Shutdown()
  118. foundPaths := finder.FindMatching(root,
  119. func(entries DirEntries) (dirs []string, files []string) {
  120. matches := []string{}
  121. for _, foundName := range entries.FileNames {
  122. if foundName == "findme.txt" || strings.HasSuffix(foundName, ".findme_ext") {
  123. matches = append(matches, foundName)
  124. }
  125. }
  126. return entries.DirNames, matches
  127. })
  128. absoluteMatches := []string{}
  129. for i := range expectedMatches {
  130. absoluteMatches = append(absoluteMatches, filepath.Join(root, expectedMatches[i]))
  131. }
  132. fs.AssertSameResponse(t, foundPaths, absoluteMatches)
  133. }
  134. // testAgainstSeveralThreadcounts runs the given test for each threadcount that we care to test
  135. func testAgainstSeveralThreadcounts(t *testing.T, tester func(t *testing.T, numThreads int)) {
  136. // test singlethreaded, multithreaded, and also using the same number of threads as
  137. // will be used on the current system
  138. threadCounts := []int{1, 2, defaultNumThreads}
  139. for _, numThreads := range threadCounts {
  140. testName := fmt.Sprintf("%v threads", numThreads)
  141. // store numThreads in a new variable to prevent numThreads from changing in each loop
  142. localNumThreads := numThreads
  143. t.Run(testName, func(t *testing.T) {
  144. tester(t, localNumThreads)
  145. })
  146. }
  147. }
  148. // end of utils, start of individual tests
  149. func TestSingleFile(t *testing.T) {
  150. runSimpleTest(t,
  151. []string{"findme.txt"},
  152. []string{"findme.txt"},
  153. )
  154. }
  155. func TestIncludeFiles(t *testing.T) {
  156. runSimpleTest(t,
  157. []string{"findme.txt", "skipme.txt"},
  158. []string{"findme.txt"},
  159. )
  160. }
  161. func TestIncludeFilesAndSuffixes(t *testing.T) {
  162. runTestWithSuffixes(t,
  163. []string{"findme.txt", "skipme.txt", "alsome.findme_ext"},
  164. []string{"findme.txt", "alsome.findme_ext"},
  165. )
  166. }
  167. func TestNestedDirectories(t *testing.T) {
  168. runSimpleTest(t,
  169. []string{"findme.txt", "skipme.txt", "subdir/findme.txt", "subdir/skipme.txt"},
  170. []string{"findme.txt", "subdir/findme.txt"},
  171. )
  172. }
  173. func TestNestedDirectoriesWithSuffixes(t *testing.T) {
  174. runTestWithSuffixes(t,
  175. []string{"findme.txt", "skipme.txt", "subdir/findme.txt", "subdir/skipme.txt", "subdir/alsome.findme_ext"},
  176. []string{"findme.txt", "subdir/findme.txt", "subdir/alsome.findme_ext"},
  177. )
  178. }
  179. func TestEmptyDirectory(t *testing.T) {
  180. runSimpleTest(t,
  181. []string{},
  182. []string{},
  183. )
  184. }
  185. func TestEmptyPath(t *testing.T) {
  186. filesystem := newFs()
  187. root := "/tmp"
  188. fs.Create(t, filepath.Join(root, "findme.txt"), filesystem)
  189. finder := newFinder(
  190. t,
  191. filesystem,
  192. CacheParams{
  193. RootDirs: []string{root},
  194. IncludeFiles: []string{"findme.txt", "skipme.txt"},
  195. },
  196. )
  197. defer finder.Shutdown()
  198. foundPaths := finder.FindNamedAt("", "findme.txt")
  199. fs.AssertSameResponse(t, foundPaths, []string{})
  200. }
  201. func TestFilesystemRoot(t *testing.T) {
  202. testWithNumThreads := func(t *testing.T, numThreads int) {
  203. filesystem := newFs()
  204. root := "/"
  205. createdPath := "/findme.txt"
  206. fs.Create(t, createdPath, filesystem)
  207. finder := newFinderWithNumThreads(
  208. t,
  209. filesystem,
  210. CacheParams{
  211. RootDirs: []string{root},
  212. IncludeFiles: []string{"findme.txt", "skipme.txt"},
  213. },
  214. numThreads,
  215. )
  216. defer finder.Shutdown()
  217. foundPaths := finder.FindNamedAt(root, "findme.txt")
  218. fs.AssertSameResponse(t, foundPaths, []string{createdPath})
  219. }
  220. testAgainstSeveralThreadcounts(t, testWithNumThreads)
  221. }
  222. func TestNonexistentDir(t *testing.T) {
  223. filesystem := newFs()
  224. fs.Create(t, "/tmp/findme.txt", filesystem)
  225. _, err := newFinderAndErr(
  226. t,
  227. filesystem,
  228. CacheParams{
  229. RootDirs: []string{"/tmp/IDontExist"},
  230. IncludeFiles: []string{"findme.txt", "skipme.txt"},
  231. },
  232. 1,
  233. )
  234. if err == nil {
  235. t.Fatal("Did not fail when given a nonexistent root directory")
  236. }
  237. }
  238. func TestExcludeDirs(t *testing.T) {
  239. filesystem := newFs()
  240. fs.Create(t, "/tmp/exclude/findme.txt", filesystem)
  241. fs.Create(t, "/tmp/exclude/subdir/findme.txt", filesystem)
  242. fs.Create(t, "/tmp/subdir/exclude/findme.txt", filesystem)
  243. fs.Create(t, "/tmp/subdir/subdir/findme.txt", filesystem)
  244. fs.Create(t, "/tmp/subdir/findme.txt", filesystem)
  245. fs.Create(t, "/tmp/findme.txt", filesystem)
  246. finder := newFinder(
  247. t,
  248. filesystem,
  249. CacheParams{
  250. RootDirs: []string{"/tmp"},
  251. ExcludeDirs: []string{"exclude"},
  252. IncludeFiles: []string{"findme.txt", "skipme.txt"},
  253. },
  254. )
  255. defer finder.Shutdown()
  256. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  257. fs.AssertSameResponse(t, foundPaths,
  258. []string{"/tmp/findme.txt",
  259. "/tmp/subdir/findme.txt",
  260. "/tmp/subdir/subdir/findme.txt"})
  261. }
  262. func TestPruneFiles(t *testing.T) {
  263. filesystem := newFs()
  264. fs.Create(t, "/tmp/out/findme.txt", filesystem)
  265. fs.Create(t, "/tmp/out/.ignore-out-dir", filesystem)
  266. fs.Create(t, "/tmp/out/child/findme.txt", filesystem)
  267. fs.Create(t, "/tmp/out2/.ignore-out-dir", filesystem)
  268. fs.Create(t, "/tmp/out2/sub/findme.txt", filesystem)
  269. fs.Create(t, "/tmp/findme.txt", filesystem)
  270. fs.Create(t, "/tmp/include/findme.txt", filesystem)
  271. finder := newFinder(
  272. t,
  273. filesystem,
  274. CacheParams{
  275. RootDirs: []string{"/tmp"},
  276. PruneFiles: []string{".ignore-out-dir"},
  277. IncludeFiles: []string{"findme.txt"},
  278. },
  279. )
  280. defer finder.Shutdown()
  281. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  282. fs.AssertSameResponse(t, foundPaths,
  283. []string{"/tmp/findme.txt",
  284. "/tmp/include/findme.txt"})
  285. }
  286. // TestRootDir tests that the value of RootDirs is used
  287. // tests of the filesystem root are in TestFilesystemRoot
  288. func TestRootDir(t *testing.T) {
  289. filesystem := newFs()
  290. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  291. fs.Create(t, "/tmp/a/subdir/findme.txt", filesystem)
  292. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  293. fs.Create(t, "/tmp/b/subdir/findme.txt", filesystem)
  294. finder := newFinder(
  295. t,
  296. filesystem,
  297. CacheParams{
  298. RootDirs: []string{"/tmp/a"},
  299. IncludeFiles: []string{"findme.txt"},
  300. },
  301. )
  302. defer finder.Shutdown()
  303. foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
  304. fs.AssertSameResponse(t, foundPaths,
  305. []string{"/tmp/a/findme.txt",
  306. "/tmp/a/subdir/findme.txt"})
  307. }
  308. func TestUncachedDir(t *testing.T) {
  309. filesystem := newFs()
  310. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  311. fs.Create(t, "/tmp/a/subdir/findme.txt", filesystem)
  312. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  313. fs.Create(t, "/tmp/b/subdir/findme.txt", filesystem)
  314. finder := newFinder(
  315. t,
  316. filesystem,
  317. CacheParams{
  318. RootDirs: []string{"/tmp/b"},
  319. IncludeFiles: []string{"findme.txt"},
  320. },
  321. )
  322. foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
  323. // If the caller queries for a file that is in the cache, then computing the
  324. // correct answer won't be fast, and it would be easy for the caller to
  325. // fail to notice its slowness. Instead, we only ever search the cache for files
  326. // to return, which enforces that we can determine which files will be
  327. // interesting upfront.
  328. fs.AssertSameResponse(t, foundPaths, []string{})
  329. finder.Shutdown()
  330. }
  331. func TestSearchingForFilesExcludedFromCache(t *testing.T) {
  332. // setup filesystem
  333. filesystem := newFs()
  334. fs.Create(t, "/tmp/findme.txt", filesystem)
  335. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  336. fs.Create(t, "/tmp/a/misc.txt", filesystem)
  337. // set up the finder and run it
  338. finder := newFinder(
  339. t,
  340. filesystem,
  341. CacheParams{
  342. RootDirs: []string{"/tmp"},
  343. IncludeFiles: []string{"findme.txt"},
  344. },
  345. )
  346. foundPaths := finder.FindNamedAt("/tmp", "misc.txt")
  347. // If the caller queries for a file that is in the cache, then computing the
  348. // correct answer won't be fast, and it would be easy for the caller to
  349. // fail to notice its slowness. Instead, we only ever search the cache for files
  350. // to return, which enforces that we can determine which files will be
  351. // interesting upfront.
  352. fs.AssertSameResponse(t, foundPaths, []string{})
  353. finder.Shutdown()
  354. }
  355. func TestRelativeFilePaths(t *testing.T) {
  356. filesystem := newFs()
  357. fs.Create(t, "/tmp/ignore/hi.txt", filesystem)
  358. fs.Create(t, "/tmp/include/hi.txt", filesystem)
  359. fs.Create(t, "/cwd/hi.txt", filesystem)
  360. fs.Create(t, "/cwd/a/hi.txt", filesystem)
  361. fs.Create(t, "/cwd/a/a/hi.txt", filesystem)
  362. fs.Create(t, "/rel/a/hi.txt", filesystem)
  363. finder := newFinder(
  364. t,
  365. filesystem,
  366. CacheParams{
  367. RootDirs: []string{"/cwd", "../rel", "/tmp/include"},
  368. IncludeFiles: []string{"hi.txt"},
  369. },
  370. )
  371. defer finder.Shutdown()
  372. foundPaths := finder.FindNamedAt("a", "hi.txt")
  373. fs.AssertSameResponse(t, foundPaths,
  374. []string{"a/hi.txt",
  375. "a/a/hi.txt"})
  376. foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
  377. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
  378. foundPaths = finder.FindNamedAt(".", "hi.txt")
  379. fs.AssertSameResponse(t, foundPaths,
  380. []string{"hi.txt",
  381. "a/hi.txt",
  382. "a/a/hi.txt"})
  383. foundPaths = finder.FindNamedAt("/rel", "hi.txt")
  384. fs.AssertSameResponse(t, foundPaths,
  385. []string{"/rel/a/hi.txt"})
  386. foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
  387. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
  388. }
  389. // have to run this test with the race-detector (`go test -race src/android/soong/finder/*.go`)
  390. // for there to be much chance of the test actually detecting any error that may be present
  391. func TestRootDirsContainedInOtherRootDirs(t *testing.T) {
  392. filesystem := newFs()
  393. fs.Create(t, "/tmp/a/b/c/d/e/f/g/h/i/j/findme.txt", filesystem)
  394. finder := newFinder(
  395. t,
  396. filesystem,
  397. CacheParams{
  398. RootDirs: []string{"/", "/tmp/a/b/c", "/tmp/a/b/c/d/e/f", "/tmp/a/b/c/d/e/f/g/h/i"},
  399. IncludeFiles: []string{"findme.txt"},
  400. },
  401. )
  402. defer finder.Shutdown()
  403. foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
  404. fs.AssertSameResponse(t, foundPaths,
  405. []string{"/tmp/a/b/c/d/e/f/g/h/i/j/findme.txt"})
  406. }
  407. func TestFindFirst(t *testing.T) {
  408. filesystem := newFs()
  409. fs.Create(t, "/tmp/a/hi.txt", filesystem)
  410. fs.Create(t, "/tmp/b/hi.txt", filesystem)
  411. fs.Create(t, "/tmp/b/a/hi.txt", filesystem)
  412. finder := newFinder(
  413. t,
  414. filesystem,
  415. CacheParams{
  416. RootDirs: []string{"/tmp"},
  417. IncludeFiles: []string{"hi.txt"},
  418. },
  419. )
  420. defer finder.Shutdown()
  421. foundPaths := finder.FindFirstNamed("hi.txt")
  422. fs.AssertSameResponse(t, foundPaths,
  423. []string{"/tmp/a/hi.txt",
  424. "/tmp/b/hi.txt"},
  425. )
  426. }
  427. func TestConcurrentFindSameDirectory(t *testing.T) {
  428. testWithNumThreads := func(t *testing.T, numThreads int) {
  429. filesystem := newFs()
  430. // create a bunch of files and directories
  431. paths := []string{}
  432. for i := 0; i < 10; i++ {
  433. parentDir := fmt.Sprintf("/tmp/%v", i)
  434. for j := 0; j < 10; j++ {
  435. filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
  436. paths = append(paths, filePath)
  437. }
  438. }
  439. sort.Strings(paths)
  440. for _, path := range paths {
  441. fs.Create(t, path, filesystem)
  442. }
  443. // set up a finder
  444. finder := newFinderWithNumThreads(
  445. t,
  446. filesystem,
  447. CacheParams{
  448. RootDirs: []string{"/tmp"},
  449. IncludeFiles: []string{"findme.txt"},
  450. },
  451. numThreads,
  452. )
  453. defer finder.Shutdown()
  454. numTests := 20
  455. results := make(chan []string, numTests)
  456. // make several parallel calls to the finder
  457. for i := 0; i < numTests; i++ {
  458. go func() {
  459. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  460. results <- foundPaths
  461. }()
  462. }
  463. // check that each response was correct
  464. for i := 0; i < numTests; i++ {
  465. foundPaths := <-results
  466. fs.AssertSameResponse(t, foundPaths, paths)
  467. }
  468. }
  469. testAgainstSeveralThreadcounts(t, testWithNumThreads)
  470. }
  471. func TestConcurrentFindDifferentDirectories(t *testing.T) {
  472. filesystem := newFs()
  473. // create a bunch of files and directories
  474. allFiles := []string{}
  475. numSubdirs := 10
  476. rootPaths := []string{}
  477. queryAnswers := [][]string{}
  478. for i := 0; i < numSubdirs; i++ {
  479. parentDir := fmt.Sprintf("/tmp/%v", i)
  480. rootPaths = append(rootPaths, parentDir)
  481. queryAnswers = append(queryAnswers, []string{})
  482. for j := 0; j < 10; j++ {
  483. filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
  484. queryAnswers[i] = append(queryAnswers[i], filePath)
  485. allFiles = append(allFiles, filePath)
  486. }
  487. sort.Strings(queryAnswers[i])
  488. }
  489. sort.Strings(allFiles)
  490. for _, path := range allFiles {
  491. fs.Create(t, path, filesystem)
  492. }
  493. // set up a finder
  494. finder := newFinder(
  495. t,
  496. filesystem,
  497. CacheParams{
  498. RootDirs: []string{"/tmp"},
  499. IncludeFiles: []string{"findme.txt"},
  500. },
  501. )
  502. defer finder.Shutdown()
  503. type testRun struct {
  504. path string
  505. foundMatches []string
  506. correctMatches []string
  507. }
  508. numTests := numSubdirs + 1
  509. testRuns := make(chan testRun, numTests)
  510. searchAt := func(path string, correctMatches []string) {
  511. foundPaths := finder.FindNamedAt(path, "findme.txt")
  512. testRuns <- testRun{path, foundPaths, correctMatches}
  513. }
  514. // make several parallel calls to the finder
  515. go searchAt("/tmp", allFiles)
  516. for i := 0; i < len(rootPaths); i++ {
  517. go searchAt(rootPaths[i], queryAnswers[i])
  518. }
  519. // check that each response was correct
  520. for i := 0; i < numTests; i++ {
  521. testRun := <-testRuns
  522. fs.AssertSameResponse(t, testRun.foundMatches, testRun.correctMatches)
  523. }
  524. }
  525. func TestStrangelyFormattedPaths(t *testing.T) {
  526. filesystem := newFs()
  527. fs.Create(t, "/tmp/findme.txt", filesystem)
  528. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  529. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  530. finder := newFinder(
  531. t,
  532. filesystem,
  533. CacheParams{
  534. RootDirs: []string{"//tmp//a//.."},
  535. IncludeFiles: []string{"findme.txt"},
  536. },
  537. )
  538. defer finder.Shutdown()
  539. foundPaths := finder.FindNamedAt("//tmp//a//..", "findme.txt")
  540. fs.AssertSameResponse(t, foundPaths,
  541. []string{"/tmp/a/findme.txt",
  542. "/tmp/b/findme.txt",
  543. "/tmp/findme.txt"})
  544. }
  545. func TestCorruptedCacheHeader(t *testing.T) {
  546. filesystem := newFs()
  547. fs.Create(t, "/tmp/findme.txt", filesystem)
  548. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  549. fs.Write(t, "/finder/finder-db", "sample header", filesystem)
  550. finder := newFinder(
  551. t,
  552. filesystem,
  553. CacheParams{
  554. RootDirs: []string{"/tmp"},
  555. IncludeFiles: []string{"findme.txt"},
  556. },
  557. )
  558. defer finder.Shutdown()
  559. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  560. fs.AssertSameResponse(t, foundPaths,
  561. []string{"/tmp/a/findme.txt",
  562. "/tmp/findme.txt"})
  563. }
  564. func TestCanUseCache(t *testing.T) {
  565. // setup filesystem
  566. filesystem := newFs()
  567. fs.Create(t, "/tmp/findme.txt", filesystem)
  568. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  569. // run the first finder
  570. finder := newFinder(
  571. t,
  572. filesystem,
  573. CacheParams{
  574. RootDirs: []string{"/tmp"},
  575. IncludeFiles: []string{"findme.txt"},
  576. },
  577. )
  578. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  579. // check the response of the first finder
  580. correctResponse := []string{"/tmp/a/findme.txt",
  581. "/tmp/findme.txt"}
  582. fs.AssertSameResponse(t, foundPaths, correctResponse)
  583. finder.Shutdown()
  584. // check results
  585. cacheText := fs.Read(t, finder.DbPath, filesystem)
  586. if len(cacheText) < 1 {
  587. t.Fatalf("saved cache db is empty\n")
  588. }
  589. if len(filesystem.StatCalls) == 0 {
  590. t.Fatal("No Stat calls recorded by mock filesystem")
  591. }
  592. if len(filesystem.ReadDirCalls) == 0 {
  593. t.Fatal("No ReadDir calls recorded by filesystem")
  594. }
  595. statCalls := filesystem.StatCalls
  596. filesystem.ClearMetrics()
  597. // run the second finder
  598. finder2 := finderWithSameParams(t, finder)
  599. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  600. // check results
  601. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
  602. fs.AssertSameReadDirCalls(t, filesystem.StatCalls, statCalls)
  603. finder2.Shutdown()
  604. }
  605. func TestCorruptedCacheBody(t *testing.T) {
  606. // setup filesystem
  607. filesystem := newFs()
  608. fs.Create(t, "/tmp/findme.txt", filesystem)
  609. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  610. // run the first finder
  611. finder := newFinder(
  612. t,
  613. filesystem,
  614. CacheParams{
  615. RootDirs: []string{"/tmp"},
  616. IncludeFiles: []string{"findme.txt"},
  617. },
  618. )
  619. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  620. finder.Shutdown()
  621. // check the response of the first finder
  622. correctResponse := []string{"/tmp/a/findme.txt",
  623. "/tmp/findme.txt"}
  624. fs.AssertSameResponse(t, foundPaths, correctResponse)
  625. numStatCalls := len(filesystem.StatCalls)
  626. numReadDirCalls := len(filesystem.ReadDirCalls)
  627. // load the cache file, corrupt it, and save it
  628. cacheReader, err := filesystem.Open(finder.DbPath)
  629. if err != nil {
  630. t.Fatal(err)
  631. }
  632. cacheData, err := ioutil.ReadAll(cacheReader)
  633. if err != nil {
  634. t.Fatal(err)
  635. }
  636. cacheData = append(cacheData, []byte("DontMindMe")...)
  637. filesystem.WriteFile(finder.DbPath, cacheData, 0777)
  638. filesystem.ClearMetrics()
  639. // run the second finder
  640. finder2 := finderWithSameParams(t, finder)
  641. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  642. // check results
  643. fs.AssertSameResponse(t, foundPaths, correctResponse)
  644. numNewStatCalls := len(filesystem.StatCalls)
  645. numNewReadDirCalls := len(filesystem.ReadDirCalls)
  646. // It's permissable to make more Stat calls with a corrupted cache because
  647. // the Finder may restart once it detects corruption.
  648. // However, it may have already issued many Stat calls.
  649. // Because a corrupted db is not expected to be a common (or even a supported case),
  650. // we don't care to optimize it and don't cache the already-issued Stat calls
  651. if numNewReadDirCalls < numReadDirCalls {
  652. t.Fatalf(
  653. "Finder made fewer ReadDir calls with a corrupted cache (%v calls) than with no cache"+
  654. " (%v calls)",
  655. numNewReadDirCalls, numReadDirCalls)
  656. }
  657. if numNewStatCalls < numStatCalls {
  658. t.Fatalf(
  659. "Finder made fewer Stat calls with a corrupted cache (%v calls) than with no cache (%v calls)",
  660. numNewStatCalls, numStatCalls)
  661. }
  662. finder2.Shutdown()
  663. }
  664. func TestStatCalls(t *testing.T) {
  665. // setup filesystem
  666. filesystem := newFs()
  667. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  668. // run finder
  669. finder := newFinder(
  670. t,
  671. filesystem,
  672. CacheParams{
  673. RootDirs: []string{"/tmp"},
  674. IncludeFiles: []string{"findme.txt"},
  675. },
  676. )
  677. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  678. finder.Shutdown()
  679. // check response
  680. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
  681. fs.AssertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a"})
  682. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a"})
  683. }
  684. func TestFileAdded(t *testing.T) {
  685. // setup filesystem
  686. filesystem := newFs()
  687. fs.Create(t, "/tmp/ignoreme.txt", filesystem)
  688. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  689. fs.Create(t, "/tmp/b/ignore.txt", filesystem)
  690. fs.Create(t, "/tmp/b/c/nope.txt", filesystem)
  691. fs.Create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
  692. // run the first finder
  693. finder := newFinder(
  694. t,
  695. filesystem,
  696. CacheParams{
  697. RootDirs: []string{"/tmp"},
  698. IncludeFiles: []string{"findme.txt"},
  699. },
  700. )
  701. filesystem.Clock.Tick()
  702. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  703. finder.Shutdown()
  704. // check the response of the first finder
  705. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
  706. // modify the filesystem
  707. filesystem.Clock.Tick()
  708. fs.Create(t, "/tmp/b/c/findme.txt", filesystem)
  709. filesystem.Clock.Tick()
  710. filesystem.ClearMetrics()
  711. // run the second finder
  712. finder2 := finderWithSameParams(t, finder)
  713. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  714. // check results
  715. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt", "/tmp/b/c/findme.txt"})
  716. fs.AssertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d"})
  717. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b/c"})
  718. finder2.Shutdown()
  719. }
  720. func TestDirectoriesAdded(t *testing.T) {
  721. // setup filesystem
  722. filesystem := newFs()
  723. fs.Create(t, "/tmp/ignoreme.txt", filesystem)
  724. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  725. fs.Create(t, "/tmp/b/ignore.txt", filesystem)
  726. fs.Create(t, "/tmp/b/c/nope.txt", filesystem)
  727. fs.Create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
  728. // run the first finder
  729. finder := newFinder(
  730. t,
  731. filesystem,
  732. CacheParams{
  733. RootDirs: []string{"/tmp"},
  734. IncludeFiles: []string{"findme.txt"},
  735. },
  736. )
  737. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  738. finder.Shutdown()
  739. // check the response of the first finder
  740. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
  741. // modify the filesystem
  742. filesystem.Clock.Tick()
  743. fs.Create(t, "/tmp/b/c/new/findme.txt", filesystem)
  744. fs.Create(t, "/tmp/b/c/new/new2/findme.txt", filesystem)
  745. fs.Create(t, "/tmp/b/c/new/new2/ignoreme.txt", filesystem)
  746. filesystem.ClearMetrics()
  747. // run the second finder
  748. finder2 := finderWithSameParams(t, finder)
  749. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  750. // check results
  751. fs.AssertSameResponse(t, foundPaths,
  752. []string{"/tmp/a/findme.txt", "/tmp/b/c/new/findme.txt", "/tmp/b/c/new/new2/findme.txt"})
  753. fs.AssertSameStatCalls(t, filesystem.StatCalls,
  754. []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d", "/tmp/b/c/new", "/tmp/b/c/new/new2"})
  755. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b/c", "/tmp/b/c/new", "/tmp/b/c/new/new2"})
  756. finder2.Shutdown()
  757. }
  758. func TestDirectoryAndSubdirectoryBothUpdated(t *testing.T) {
  759. // setup filesystem
  760. filesystem := newFs()
  761. fs.Create(t, "/tmp/hi1.txt", filesystem)
  762. fs.Create(t, "/tmp/a/hi1.txt", filesystem)
  763. // run the first finder
  764. finder := newFinder(
  765. t,
  766. filesystem,
  767. CacheParams{
  768. RootDirs: []string{"/tmp"},
  769. IncludeFiles: []string{"hi1.txt", "hi2.txt"},
  770. },
  771. )
  772. foundPaths := finder.FindNamedAt("/tmp", "hi1.txt")
  773. finder.Shutdown()
  774. // check the response of the first finder
  775. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/hi1.txt", "/tmp/a/hi1.txt"})
  776. // modify the filesystem
  777. filesystem.Clock.Tick()
  778. fs.Create(t, "/tmp/hi2.txt", filesystem)
  779. fs.Create(t, "/tmp/a/hi2.txt", filesystem)
  780. filesystem.ClearMetrics()
  781. // run the second finder
  782. finder2 := finderWithSameParams(t, finder)
  783. foundPaths = finder2.FindAll()
  784. // check results
  785. fs.AssertSameResponse(t, foundPaths,
  786. []string{"/tmp/hi1.txt", "/tmp/hi2.txt", "/tmp/a/hi1.txt", "/tmp/a/hi2.txt"})
  787. fs.AssertSameStatCalls(t, filesystem.StatCalls,
  788. []string{"/tmp", "/tmp/a"})
  789. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a"})
  790. finder2.Shutdown()
  791. }
  792. func TestFileDeleted(t *testing.T) {
  793. // setup filesystem
  794. filesystem := newFs()
  795. fs.Create(t, "/tmp/ignoreme.txt", filesystem)
  796. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  797. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  798. fs.Create(t, "/tmp/b/c/nope.txt", filesystem)
  799. fs.Create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
  800. // run the first finder
  801. finder := newFinder(
  802. t,
  803. filesystem,
  804. CacheParams{
  805. RootDirs: []string{"/tmp"},
  806. IncludeFiles: []string{"findme.txt"},
  807. },
  808. )
  809. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  810. finder.Shutdown()
  811. // check the response of the first finder
  812. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt", "/tmp/b/findme.txt"})
  813. // modify the filesystem
  814. filesystem.Clock.Tick()
  815. fs.Delete(t, "/tmp/b/findme.txt", filesystem)
  816. filesystem.ClearMetrics()
  817. // run the second finder
  818. finder2 := finderWithSameParams(t, finder)
  819. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  820. // check results
  821. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
  822. fs.AssertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d"})
  823. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b"})
  824. finder2.Shutdown()
  825. }
  826. func TestDirectoriesDeleted(t *testing.T) {
  827. // setup filesystem
  828. filesystem := newFs()
  829. fs.Create(t, "/tmp/findme.txt", filesystem)
  830. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  831. fs.Create(t, "/tmp/a/1/findme.txt", filesystem)
  832. fs.Create(t, "/tmp/a/1/2/findme.txt", filesystem)
  833. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  834. // run the first finder
  835. finder := newFinder(
  836. t,
  837. filesystem,
  838. CacheParams{
  839. RootDirs: []string{"/tmp"},
  840. IncludeFiles: []string{"findme.txt"},
  841. },
  842. )
  843. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  844. finder.Shutdown()
  845. // check the response of the first finder
  846. fs.AssertSameResponse(t, foundPaths,
  847. []string{"/tmp/findme.txt",
  848. "/tmp/a/findme.txt",
  849. "/tmp/a/1/findme.txt",
  850. "/tmp/a/1/2/findme.txt",
  851. "/tmp/b/findme.txt"})
  852. // modify the filesystem
  853. filesystem.Clock.Tick()
  854. fs.RemoveAll(t, "/tmp/a/1", filesystem)
  855. filesystem.ClearMetrics()
  856. // run the second finder
  857. finder2 := finderWithSameParams(t, finder)
  858. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  859. // check results
  860. fs.AssertSameResponse(t, foundPaths,
  861. []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/b/findme.txt"})
  862. // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
  863. // if the Finder detects the nonexistence of /tmp/a/1
  864. // However, when resuming from cache, we don't want the Finder to necessarily wait
  865. // to stat a directory until after statting its parent.
  866. // So here we just include /tmp/a/1/2 in the list.
  867. // The Finder is currently implemented to always restat every dir and
  868. // to not short-circuit due to nonexistence of parents (but it will remove
  869. // missing dirs from the cache for next time)
  870. fs.AssertSameStatCalls(t, filesystem.StatCalls,
  871. []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b"})
  872. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/a"})
  873. finder2.Shutdown()
  874. }
  875. func TestDirectoriesMoved(t *testing.T) {
  876. // setup filesystem
  877. filesystem := newFs()
  878. fs.Create(t, "/tmp/findme.txt", filesystem)
  879. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  880. fs.Create(t, "/tmp/a/1/findme.txt", filesystem)
  881. fs.Create(t, "/tmp/a/1/2/findme.txt", filesystem)
  882. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  883. // run the first finder
  884. finder := newFinder(
  885. t,
  886. filesystem,
  887. CacheParams{
  888. RootDirs: []string{"/tmp"},
  889. IncludeFiles: []string{"findme.txt"},
  890. },
  891. )
  892. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  893. finder.Shutdown()
  894. // check the response of the first finder
  895. fs.AssertSameResponse(t, foundPaths,
  896. []string{"/tmp/findme.txt",
  897. "/tmp/a/findme.txt",
  898. "/tmp/a/1/findme.txt",
  899. "/tmp/a/1/2/findme.txt",
  900. "/tmp/b/findme.txt"})
  901. // modify the filesystem
  902. filesystem.Clock.Tick()
  903. fs.Move(t, "/tmp/a", "/tmp/c", filesystem)
  904. filesystem.ClearMetrics()
  905. // run the second finder
  906. finder2 := finderWithSameParams(t, finder)
  907. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  908. // check results
  909. fs.AssertSameResponse(t, foundPaths,
  910. []string{"/tmp/findme.txt",
  911. "/tmp/b/findme.txt",
  912. "/tmp/c/findme.txt",
  913. "/tmp/c/1/findme.txt",
  914. "/tmp/c/1/2/findme.txt"})
  915. // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
  916. // if the Finder detects the nonexistence of /tmp/a/1
  917. // However, when resuming from cache, we don't want the Finder to necessarily wait
  918. // to stat a directory until after statting its parent.
  919. // So here we just include /tmp/a/1/2 in the list.
  920. // The Finder is currently implemented to always restat every dir and
  921. // to not short-circuit due to nonexistence of parents (but it will remove
  922. // missing dirs from the cache for next time)
  923. fs.AssertSameStatCalls(t, filesystem.StatCalls,
  924. []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b", "/tmp/c", "/tmp/c/1", "/tmp/c/1/2"})
  925. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/c", "/tmp/c/1", "/tmp/c/1/2"})
  926. finder2.Shutdown()
  927. }
  928. func TestDirectoriesSwapped(t *testing.T) {
  929. // setup filesystem
  930. filesystem := newFs()
  931. fs.Create(t, "/tmp/findme.txt", filesystem)
  932. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  933. fs.Create(t, "/tmp/a/1/findme.txt", filesystem)
  934. fs.Create(t, "/tmp/a/1/2/findme.txt", filesystem)
  935. fs.Create(t, "/tmp/b/findme.txt", filesystem)
  936. // run the first finder
  937. finder := newFinder(
  938. t,
  939. filesystem,
  940. CacheParams{
  941. RootDirs: []string{"/tmp"},
  942. IncludeFiles: []string{"findme.txt"},
  943. },
  944. )
  945. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  946. finder.Shutdown()
  947. // check the response of the first finder
  948. fs.AssertSameResponse(t, foundPaths,
  949. []string{"/tmp/findme.txt",
  950. "/tmp/a/findme.txt",
  951. "/tmp/a/1/findme.txt",
  952. "/tmp/a/1/2/findme.txt",
  953. "/tmp/b/findme.txt"})
  954. // modify the filesystem
  955. filesystem.Clock.Tick()
  956. fs.Move(t, "/tmp/a", "/tmp/temp", filesystem)
  957. fs.Move(t, "/tmp/b", "/tmp/a", filesystem)
  958. fs.Move(t, "/tmp/temp", "/tmp/b", filesystem)
  959. filesystem.ClearMetrics()
  960. // run the second finder
  961. finder2 := finderWithSameParams(t, finder)
  962. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  963. // check results
  964. fs.AssertSameResponse(t, foundPaths,
  965. []string{"/tmp/findme.txt",
  966. "/tmp/a/findme.txt",
  967. "/tmp/b/findme.txt",
  968. "/tmp/b/1/findme.txt",
  969. "/tmp/b/1/2/findme.txt"})
  970. // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
  971. // if the Finder detects the nonexistence of /tmp/a/1
  972. // However, when resuming from cache, we don't want the Finder to necessarily wait
  973. // to stat a directory until after statting its parent.
  974. // So here we just include /tmp/a/1/2 in the list.
  975. // The Finder is currently implemented to always restat every dir and
  976. // to not short-circuit due to nonexistence of parents (but it will remove
  977. // missing dirs from the cache for next time)
  978. fs.AssertSameStatCalls(t, filesystem.StatCalls,
  979. []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b", "/tmp/b/1", "/tmp/b/1/2"})
  980. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/1", "/tmp/b/1/2"})
  981. finder2.Shutdown()
  982. }
  983. // runFsReplacementTest tests a change modifying properties of the filesystem itself:
  984. // runFsReplacementTest tests changing the user, the hostname, or the device number
  985. // runFsReplacementTest is a helper method called by other tests
  986. func runFsReplacementTest(t *testing.T, fs1 *fs.MockFs, fs2 *fs.MockFs) {
  987. // setup fs1
  988. fs.Create(t, "/tmp/findme.txt", fs1)
  989. fs.Create(t, "/tmp/a/findme.txt", fs1)
  990. fs.Create(t, "/tmp/a/a/findme.txt", fs1)
  991. // setup fs2 to have the same directories but different files
  992. fs.Create(t, "/tmp/findme.txt", fs2)
  993. fs.Create(t, "/tmp/a/findme.txt", fs2)
  994. fs.Create(t, "/tmp/a/a/ignoreme.txt", fs2)
  995. fs.Create(t, "/tmp/a/b/findme.txt", fs2)
  996. // run the first finder
  997. finder := newFinder(
  998. t,
  999. fs1,
  1000. CacheParams{
  1001. RootDirs: []string{"/tmp"},
  1002. IncludeFiles: []string{"findme.txt"},
  1003. },
  1004. )
  1005. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  1006. finder.Shutdown()
  1007. // check the response of the first finder
  1008. fs.AssertSameResponse(t, foundPaths,
  1009. []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/a/a/findme.txt"})
  1010. // copy the cache data from the first filesystem to the second
  1011. cacheContent := fs.Read(t, finder.DbPath, fs1)
  1012. fs.Write(t, finder.DbPath, cacheContent, fs2)
  1013. // run the second finder, with the same config and same cache contents but a different filesystem
  1014. finder2 := newFinder(
  1015. t,
  1016. fs2,
  1017. CacheParams{
  1018. RootDirs: []string{"/tmp"},
  1019. IncludeFiles: []string{"findme.txt"},
  1020. },
  1021. )
  1022. foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
  1023. // check results
  1024. fs.AssertSameResponse(t, foundPaths,
  1025. []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/a/b/findme.txt"})
  1026. fs.AssertSameStatCalls(t, fs2.StatCalls,
  1027. []string{"/tmp", "/tmp/a", "/tmp/a/a", "/tmp/a/b"})
  1028. fs.AssertSameReadDirCalls(t, fs2.ReadDirCalls,
  1029. []string{"/tmp", "/tmp/a", "/tmp/a/a", "/tmp/a/b"})
  1030. finder2.Shutdown()
  1031. }
  1032. func TestChangeOfDevice(t *testing.T) {
  1033. fs1 := newFs()
  1034. // not as fine-grained mounting controls as a real filesystem, but should be adequate
  1035. fs1.SetDeviceNumber(0)
  1036. fs2 := newFs()
  1037. fs2.SetDeviceNumber(1)
  1038. runFsReplacementTest(t, fs1, fs2)
  1039. }
  1040. func TestChangeOfUserOrHost(t *testing.T) {
  1041. fs1 := newFs()
  1042. fs1.SetViewId("me@here")
  1043. fs2 := newFs()
  1044. fs2.SetViewId("you@there")
  1045. runFsReplacementTest(t, fs1, fs2)
  1046. }
  1047. func TestConsistentCacheOrdering(t *testing.T) {
  1048. // setup filesystem
  1049. filesystem := newFs()
  1050. for i := 0; i < 5; i++ {
  1051. fs.Create(t, fmt.Sprintf("/tmp/%v/findme.txt", i), filesystem)
  1052. }
  1053. // run the first finder
  1054. finder := newFinder(
  1055. t,
  1056. filesystem,
  1057. CacheParams{
  1058. RootDirs: []string{"/tmp"},
  1059. IncludeFiles: []string{"findme.txt"},
  1060. },
  1061. )
  1062. finder.FindNamedAt("/tmp", "findme.txt")
  1063. finder.Shutdown()
  1064. // read db file
  1065. string1 := fs.Read(t, finder.DbPath, filesystem)
  1066. err := filesystem.Remove(finder.DbPath)
  1067. if err != nil {
  1068. t.Fatal(err)
  1069. }
  1070. // run another finder
  1071. finder2 := finderWithSameParams(t, finder)
  1072. finder2.FindNamedAt("/tmp", "findme.txt")
  1073. finder2.Shutdown()
  1074. string2 := fs.Read(t, finder.DbPath, filesystem)
  1075. if string1 != string2 {
  1076. t.Errorf("Running Finder twice generated two dbs not having identical contents.\n"+
  1077. "Content of first file:\n"+
  1078. "\n"+
  1079. "%v"+
  1080. "\n"+
  1081. "\n"+
  1082. "Content of second file:\n"+
  1083. "\n"+
  1084. "%v\n"+
  1085. "\n",
  1086. string1,
  1087. string2,
  1088. )
  1089. }
  1090. }
  1091. func TestNumSyscallsOfSecondFind(t *testing.T) {
  1092. // setup filesystem
  1093. filesystem := newFs()
  1094. fs.Create(t, "/tmp/findme.txt", filesystem)
  1095. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  1096. fs.Create(t, "/tmp/a/misc.txt", filesystem)
  1097. // set up the finder and run it once
  1098. finder := newFinder(
  1099. t,
  1100. filesystem,
  1101. CacheParams{
  1102. RootDirs: []string{"/tmp"},
  1103. IncludeFiles: []string{"findme.txt"},
  1104. },
  1105. )
  1106. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  1107. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/findme.txt", "/tmp/a/findme.txt"})
  1108. filesystem.ClearMetrics()
  1109. // run the finder again and confirm it doesn't check the filesystem
  1110. refoundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  1111. fs.AssertSameResponse(t, refoundPaths, foundPaths)
  1112. fs.AssertSameStatCalls(t, filesystem.StatCalls, []string{})
  1113. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
  1114. finder.Shutdown()
  1115. }
  1116. func TestChangingParamsOfSecondFind(t *testing.T) {
  1117. // setup filesystem
  1118. filesystem := newFs()
  1119. fs.Create(t, "/tmp/findme.txt", filesystem)
  1120. fs.Create(t, "/tmp/a/findme.txt", filesystem)
  1121. fs.Create(t, "/tmp/a/metoo.txt", filesystem)
  1122. // set up the finder and run it once
  1123. finder := newFinder(
  1124. t,
  1125. filesystem,
  1126. CacheParams{
  1127. RootDirs: []string{"/tmp"},
  1128. IncludeFiles: []string{"findme.txt", "metoo.txt"},
  1129. },
  1130. )
  1131. foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
  1132. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/findme.txt", "/tmp/a/findme.txt"})
  1133. filesystem.ClearMetrics()
  1134. // run the finder again and confirm it gets the right answer without asking the filesystem
  1135. refoundPaths := finder.FindNamedAt("/tmp", "metoo.txt")
  1136. fs.AssertSameResponse(t, refoundPaths, []string{"/tmp/a/metoo.txt"})
  1137. fs.AssertSameStatCalls(t, filesystem.StatCalls, []string{})
  1138. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
  1139. finder.Shutdown()
  1140. }
  1141. func TestSymlinkPointingToFile(t *testing.T) {
  1142. // setup filesystem
  1143. filesystem := newFs()
  1144. fs.Create(t, "/tmp/a/hi.txt", filesystem)
  1145. fs.Create(t, "/tmp/a/ignoreme.txt", filesystem)
  1146. fs.Link(t, "/tmp/hi.txt", "a/hi.txt", filesystem)
  1147. fs.Link(t, "/tmp/b/hi.txt", "../a/hi.txt", filesystem)
  1148. fs.Link(t, "/tmp/c/hi.txt", "/tmp/hi.txt", filesystem)
  1149. fs.Link(t, "/tmp/d/hi.txt", "../a/bye.txt", filesystem)
  1150. fs.Link(t, "/tmp/d/bye.txt", "../a/hi.txt", filesystem)
  1151. fs.Link(t, "/tmp/e/bye.txt", "../a/bye.txt", filesystem)
  1152. fs.Link(t, "/tmp/f/hi.txt", "somethingThatDoesntExist", filesystem)
  1153. // set up the finder and run it once
  1154. finder := newFinder(
  1155. t,
  1156. filesystem,
  1157. CacheParams{
  1158. RootDirs: []string{"/tmp"},
  1159. IncludeFiles: []string{"hi.txt"},
  1160. },
  1161. )
  1162. foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
  1163. // should search based on the name of the link rather than the destination or validity of the link
  1164. correctResponse := []string{
  1165. "/tmp/a/hi.txt",
  1166. "/tmp/hi.txt",
  1167. "/tmp/b/hi.txt",
  1168. "/tmp/c/hi.txt",
  1169. "/tmp/d/hi.txt",
  1170. "/tmp/f/hi.txt",
  1171. }
  1172. fs.AssertSameResponse(t, foundPaths, correctResponse)
  1173. }
  1174. func TestSymlinkPointingToDirectory(t *testing.T) {
  1175. // setup filesystem
  1176. filesystem := newFs()
  1177. fs.Create(t, "/tmp/dir/hi.txt", filesystem)
  1178. fs.Create(t, "/tmp/dir/ignoreme.txt", filesystem)
  1179. fs.Link(t, "/tmp/links/dir", "../dir", filesystem)
  1180. fs.Link(t, "/tmp/links/link", "../dir", filesystem)
  1181. fs.Link(t, "/tmp/links/hi.txt", "../dir", filesystem)
  1182. fs.Link(t, "/tmp/links/broken", "nothingHere", filesystem)
  1183. fs.Link(t, "/tmp/links/recursive", "recursive", filesystem)
  1184. // set up the finder and run it once
  1185. finder := newFinder(
  1186. t,
  1187. filesystem,
  1188. CacheParams{
  1189. RootDirs: []string{"/tmp"},
  1190. IncludeFiles: []string{"hi.txt"},
  1191. },
  1192. )
  1193. foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
  1194. // should completely ignore symlinks that point to directories
  1195. correctResponse := []string{
  1196. "/tmp/dir/hi.txt",
  1197. }
  1198. fs.AssertSameResponse(t, foundPaths, correctResponse)
  1199. }
  1200. // TestAddPruneFile confirms that adding a prune-file (into a directory for which we
  1201. // already had a cache) causes the directory to be ignored
  1202. func TestAddPruneFile(t *testing.T) {
  1203. // setup filesystem
  1204. filesystem := newFs()
  1205. fs.Create(t, "/tmp/out/hi.txt", filesystem)
  1206. fs.Create(t, "/tmp/out/a/hi.txt", filesystem)
  1207. fs.Create(t, "/tmp/hi.txt", filesystem)
  1208. // do find
  1209. finder := newFinder(
  1210. t,
  1211. filesystem,
  1212. CacheParams{
  1213. RootDirs: []string{"/tmp"},
  1214. PruneFiles: []string{".ignore-out-dir"},
  1215. IncludeFiles: []string{"hi.txt"},
  1216. },
  1217. )
  1218. foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
  1219. // check result
  1220. fs.AssertSameResponse(t, foundPaths,
  1221. []string{"/tmp/hi.txt",
  1222. "/tmp/out/hi.txt",
  1223. "/tmp/out/a/hi.txt"},
  1224. )
  1225. finder.Shutdown()
  1226. // modify filesystem
  1227. filesystem.Clock.Tick()
  1228. fs.Create(t, "/tmp/out/.ignore-out-dir", filesystem)
  1229. // run another find and check its result
  1230. finder2 := finderWithSameParams(t, finder)
  1231. foundPaths = finder2.FindNamedAt("/tmp", "hi.txt")
  1232. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"})
  1233. finder2.Shutdown()
  1234. }
  1235. func TestUpdatingDbIffChanged(t *testing.T) {
  1236. // setup filesystem
  1237. filesystem := newFs()
  1238. fs.Create(t, "/tmp/a/hi.txt", filesystem)
  1239. fs.Create(t, "/tmp/b/bye.txt", filesystem)
  1240. // run the first finder
  1241. finder := newFinder(
  1242. t,
  1243. filesystem,
  1244. CacheParams{
  1245. RootDirs: []string{"/tmp"},
  1246. IncludeFiles: []string{"hi.txt"},
  1247. },
  1248. )
  1249. filesystem.Clock.Tick()
  1250. foundPaths := finder.FindAll()
  1251. finder.Shutdown()
  1252. // check results
  1253. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"})
  1254. // modify the filesystem
  1255. filesystem.Clock.Tick()
  1256. fs.Create(t, "/tmp/b/hi.txt", filesystem)
  1257. filesystem.Clock.Tick()
  1258. filesystem.ClearMetrics()
  1259. // run the second finder
  1260. finder2 := finderWithSameParams(t, finder)
  1261. foundPaths = finder2.FindAll()
  1262. finder2.Shutdown()
  1263. // check results
  1264. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt", "/tmp/b/hi.txt"})
  1265. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b"})
  1266. expectedDbWriteTime := filesystem.Clock.Time()
  1267. actualDbWriteTime := fs.ModTime(t, finder2.DbPath, filesystem)
  1268. if actualDbWriteTime != expectedDbWriteTime {
  1269. t.Fatalf("Expected to write db at %v, actually wrote db at %v\n",
  1270. expectedDbWriteTime, actualDbWriteTime)
  1271. }
  1272. // reset metrics
  1273. filesystem.ClearMetrics()
  1274. // run the third finder
  1275. finder3 := finderWithSameParams(t, finder2)
  1276. foundPaths = finder3.FindAll()
  1277. // check results
  1278. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt", "/tmp/b/hi.txt"})
  1279. fs.AssertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
  1280. finder3.Shutdown()
  1281. actualDbWriteTime = fs.ModTime(t, finder3.DbPath, filesystem)
  1282. if actualDbWriteTime != expectedDbWriteTime {
  1283. t.Fatalf("Re-wrote db even when contents did not change")
  1284. }
  1285. }
  1286. func TestDirectoryNotPermitted(t *testing.T) {
  1287. // setup filesystem
  1288. filesystem := newFs()
  1289. fs.Create(t, "/tmp/hi.txt", filesystem)
  1290. fs.Create(t, "/tmp/a/hi.txt", filesystem)
  1291. fs.Create(t, "/tmp/a/a/hi.txt", filesystem)
  1292. fs.Create(t, "/tmp/b/hi.txt", filesystem)
  1293. // run the first finder
  1294. finder := newFinder(
  1295. t,
  1296. filesystem,
  1297. CacheParams{
  1298. RootDirs: []string{"/tmp"},
  1299. IncludeFiles: []string{"hi.txt"},
  1300. },
  1301. )
  1302. filesystem.Clock.Tick()
  1303. foundPaths := finder.FindAll()
  1304. finder.Shutdown()
  1305. allPaths := []string{"/tmp/hi.txt", "/tmp/a/hi.txt", "/tmp/a/a/hi.txt", "/tmp/b/hi.txt"}
  1306. // check results
  1307. fs.AssertSameResponse(t, foundPaths, allPaths)
  1308. // modify the filesystem
  1309. filesystem.Clock.Tick()
  1310. fs.SetReadable(t, "/tmp/a", false, filesystem)
  1311. filesystem.Clock.Tick()
  1312. // run the second finder
  1313. finder2 := finderWithSameParams(t, finder)
  1314. foundPaths = finder2.FindAll()
  1315. finder2.Shutdown()
  1316. // check results
  1317. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/hi.txt", "/tmp/b/hi.txt"})
  1318. // modify the filesystem back
  1319. fs.SetReadable(t, "/tmp/a", true, filesystem)
  1320. // run the third finder
  1321. finder3 := finderWithSameParams(t, finder2)
  1322. foundPaths = finder3.FindAll()
  1323. finder3.Shutdown()
  1324. // check results
  1325. fs.AssertSameResponse(t, foundPaths, allPaths)
  1326. }
  1327. func TestFileNotPermitted(t *testing.T) {
  1328. // setup filesystem
  1329. filesystem := newFs()
  1330. fs.Create(t, "/tmp/hi.txt", filesystem)
  1331. fs.SetReadable(t, "/tmp/hi.txt", false, filesystem)
  1332. // run the first finder
  1333. finder := newFinder(
  1334. t,
  1335. filesystem,
  1336. CacheParams{
  1337. RootDirs: []string{"/tmp"},
  1338. IncludeFiles: []string{"hi.txt"},
  1339. },
  1340. )
  1341. filesystem.Clock.Tick()
  1342. foundPaths := finder.FindAll()
  1343. finder.Shutdown()
  1344. // check results
  1345. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"})
  1346. }
  1347. func TestCacheEntryPathUnexpectedError(t *testing.T) {
  1348. // setup filesystem
  1349. filesystem := newFs()
  1350. fs.Create(t, "/tmp/a/hi.txt", filesystem)
  1351. // run the first finder
  1352. finder := newFinder(
  1353. t,
  1354. filesystem,
  1355. CacheParams{
  1356. RootDirs: []string{"/tmp"},
  1357. IncludeFiles: []string{"hi.txt"},
  1358. },
  1359. )
  1360. filesystem.Clock.Tick()
  1361. foundPaths := finder.FindAll()
  1362. finder.Shutdown()
  1363. // check results
  1364. fs.AssertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"})
  1365. // make the directory not readable
  1366. fs.SetReadErr(t, "/tmp/a", os.ErrInvalid, filesystem)
  1367. // run the second finder
  1368. _, err := finderAndErrorWithSameParams(t, finder)
  1369. if err == nil {
  1370. t.Fatal("Failed to detect unexpected filesystem error")
  1371. }
  1372. }