webforms_aggregator_unittests.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298
  1. #!/usr/bin/env python
  2. # Copyright (c) 2011 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import logging
  6. import os
  7. import subprocess
  8. import tempfile
  9. import unittest
  10. # Same name as the aggregator module name.
  11. import webforms_aggregator
  12. logger = logging.getLogger(webforms_aggregator.__name__)
  13. console = logging.StreamHandler()
  14. logger.addHandler(console)
  15. # Commenting out the following line will set logger level to default: WARNING
  16. logger.setLevel(logging.INFO)
  17. class WebformsAggregatorTest(unittest.TestCase):
  18. """Unit tests for the webforms_aggregator module."""
  19. PORT1 = 8002
  20. PORT2 = 8003
  21. HOME_CONTENT = """
  22. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
  23. "http://www.w3.org/TR/html4/loose.dtd">
  24. <html>
  25. <head>
  26. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  27. <title>%s</title>
  28. </head>
  29. <body>
  30. <h1>%s</h1>
  31. <p>This is a mock site. Its mere purpose is to contribute towards testing \
  32. the aggregator crawler.</p>
  33. <ul>
  34. <li><a href="%s">page1</a></li>
  35. <li><a href="%s">page2</a></li>
  36. <li><a href="%s">page3</a></li>
  37. </ul>
  38. <hr>
  39. <p>
  40. <a href="%s">sign in</a>
  41. </p>
  42. </body>
  43. </html>
  44. """
  45. SIMPLE_PAGE_CONTENT = """
  46. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
  47. "http://www.w3.org/TR/html4/loose.dtd">
  48. <html>
  49. <head>
  50. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  51. <title>%s</title>
  52. </head>
  53. <body>
  54. <h1>%s</h1>
  55. <p>%s</p>
  56. <ul>
  57. <li><a href="%s">%s</a></li>
  58. <li><a href="%s">%s</a></li>
  59. </ul>
  60. <hr>
  61. <p>
  62. <a href="%s">return to home page</a>
  63. </p>
  64. </body>
  65. </html>
  66. """
  67. SIGNIN_CONTENT = """
  68. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
  69. "http://www.w3.org/TR/html4/loose.dtd">
  70. <html>
  71. <head>
  72. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  73. <title>%s</title>
  74. </head>
  75. <body>
  76. <h1>Sign in!</h1>
  77. <h3>%s</h3>
  78. <form>
  79. <label>User name: </label><input type="text"><br><br>
  80. <label>password: </label><input type="password"><br><br>
  81. <input type="submit" value="Sign in">
  82. </form>
  83. <hr>
  84. <p><a href="%s">return to home page</a></p>
  85. </body>
  86. </html>
  87. """
  88. REG_CONTENT = """
  89. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
  90. "http://www.w3.org/TR/html4/loose.dtd">
  91. <html>
  92. <head>
  93. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  94. <title>%s</title>
  95. </head>
  96. <body>
  97. <h1>Create a user account!</h1>
  98. <h3>Enter your data below:</h3>
  99. <form method="get">
  100. <label>First name: </label><input type="text"><br><br>
  101. <label>Surname: </label><input type="text"><br><br>
  102. <label>User name: </label><input type="text"><br><br>
  103. <label>password: </label><input type="password"><br><br>
  104. <label>retype password: </label><input type="password"><br><br>
  105. <input type="submit" value="Register">
  106. </form>
  107. <hr>
  108. <p><a href="%s">return to home page</a></p>
  109. </body>
  110. </html>
  111. """
  112. def CreateMockSiteOne(self):
  113. """Site One has a registration form.
  114. """
  115. self.files['site1_home'] = 'site1_index.html'
  116. self.files['site1_page1'] = 'site1_page1.html'
  117. self.files['site1_page2'] = 'site1_page2.html'
  118. self.files['site1_page3'] = 'site1_page3.html'
  119. self.files['site1_signin'] = 'site1_signin.html'
  120. self.files['site1_reg'] = 'site1_register.html'
  121. file_content = {}
  122. file_content[self.files['site1_home']] = self.HOME_CONTENT % (
  123. 'Site One home page', 'Welcome to site one. It has a reg page!',
  124. self.files['site1_page1'], self.files['site1_page2'],
  125. self.files['site1_page3'], self.files['site1_signin'])
  126. file_content[self.files['site1_page1']] = self.SIMPLE_PAGE_CONTENT % (
  127. 'Site One page 1',
  128. 'Page 1!', 'This is a useless page. It does almost nothing.',
  129. self.files['site1_page2'], 'page 2', self.files['site1_page3'],
  130. 'page 3', self.files['site1_home'])
  131. file_content[self.files['site1_page2']] = self.SIMPLE_PAGE_CONTENT % (
  132. 'Site One page 2', 'Page 2!',
  133. 'This is another useless page. It does almost what the page 1 does.',
  134. self.files['site1_page1'], 'page 1', self.files['site1_page3'],
  135. 'page 3', self.files['site1_home'])
  136. file_content[self.files['site1_page3']] = self.SIMPLE_PAGE_CONTENT % (
  137. 'Site One page 3', 'Page 3!',
  138. "This is the last useless page. It doesn't do anything useful at all.",
  139. self.files['site1_page1'], 'page 1', self.files['site1_page2'],
  140. 'page 2', self.files['site1_home'])
  141. file_content[self.files['site1_signin']] = self.SIGNIN_CONTENT % (
  142. 'Site One signin',
  143. 'If you don\'t have a user account click <a href="%s">here</a>.' \
  144. % self.files['site1_reg'],
  145. self.files['site1_home'])
  146. file_content[self.files['site1_reg']] = self.REG_CONTENT % (
  147. 'Site One signin', self.files['site1_home'])
  148. for filename, content in file_content.iteritems():
  149. f = open(filename, 'w')
  150. try:
  151. f.write(content)
  152. finally:
  153. f.close()
  154. def CreateMockSiteTwo(self):
  155. """ Site Two has no registration page."""
  156. self.files['site2_home'] = 'site2_index.html'
  157. self.files['site2_page1'] = 'site2_page1.html'
  158. self.files['site2_page2'] = 'site2_page2.html'
  159. self.files['site2_page3'] = 'site2_page3.html'
  160. self.files['site2_signin'] = 'site2_signin.html'
  161. file_content = {}
  162. file_content[self.files['site2_home']] = self.HOME_CONTENT % (
  163. 'Site Two home page', 'Welcome to site two. It has no reg page!',
  164. self.files['site2_page1'], self.files['site2_page2'],
  165. self.files['site2_page3'], self.files['site2_signin'])
  166. file_content[self.files['site2_page1']] = self.SIMPLE_PAGE_CONTENT % (
  167. 'Site Two page 1',
  168. 'Page 1!', 'This is a useless page. It does almost nothing.',
  169. self.files['site2_page2'], 'page 2', self.files['site2_page3'],
  170. 'page 3', self.files['site2_home'])
  171. file_content[self.files['site2_page2']] = self.SIMPLE_PAGE_CONTENT % (
  172. 'Site Two page 2', 'Page 2!',
  173. 'This is another useless page. It does almost what the page 1 does.',
  174. self.files['site2_page1'], 'page 1', self.files['site2_page3'],
  175. 'page 3', self.files['site2_home'])
  176. file_content[self.files['site2_page3']] = self.SIMPLE_PAGE_CONTENT % (
  177. 'Site Two page 3', 'Page 3!',
  178. "This is the last useless page. It doesn't do anything useful at all.",
  179. self.files['site2_page1'], 'page 1', self.files['site2_page2'],
  180. 'page 2', self.files['site2_home'])
  181. file_content[self.files['site2_signin']] = self.SIGNIN_CONTENT % (
  182. 'Site Two signin', 'You cannot register online with this site.',
  183. self.files['site2_home'])
  184. for filename, content in file_content.iteritems():
  185. f = open(filename, 'w')
  186. try:
  187. f.write(content)
  188. finally:
  189. f.close()
  190. def setUp(self):
  191. self.cwd = os.getcwdu()
  192. self.temp_dir = tempfile.mkdtemp()
  193. os.chdir(self.temp_dir)
  194. self.files = {}
  195. self.CreateMockSiteOne()
  196. self.CreateMockSiteTwo()
  197. self.files['cookie'] = 'test.cookie'
  198. self.url1 = 'http://localhost:%s/%s' % (self.PORT1,
  199. self.files['site1_home'])
  200. self.url2 = 'http://localhost:%s/%s' % (self.PORT2,
  201. self.files['site2_home'])
  202. self.domain1 = 'localhost:%s' %self.PORT1
  203. self.files['url'] = 'urls.txt'
  204. url_file_handler = open(self.files['url'], 'w')
  205. try:
  206. url_file_handler.write('URLs to crawl:')
  207. url_file_handler.write(os.linesep)
  208. for url in (self.url1, self.url2):
  209. url_file_handler.write(url)
  210. url_file_handler.write(os.linesep)
  211. finally:
  212. url_file_handler.close()
  213. command_line = 'python -u -m SimpleHTTPServer %s' % self.PORT1
  214. args = command_line.split()
  215. self.server1 = subprocess.Popen(
  216. args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  217. self.server1.stdout.readline() # Needed in order for the server to start up
  218. command_line = 'python -u -m SimpleHTTPServer %s' % self.PORT2
  219. args = command_line.split()
  220. self.server2 = subprocess.Popen(
  221. args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  222. self.server2.stdout.readline() # Needed in order for the server to start up
  223. def tearDown(self):
  224. self.server1.terminate()
  225. self.server2.terminate()
  226. for filename in self.files.values():
  227. if os.path.isfile(filename):
  228. os.unlink(filename)
  229. os.chdir(self.cwd)
  230. os.rmdir(self.temp_dir)
  231. def testRetrieverDownloadsPage(self):
  232. """Verify the retriever can download a page."""
  233. r = webforms_aggregator.Retriever(self.url1, self.domain1,
  234. self.files['cookie'])
  235. self.assertTrue(r.Download(),
  236. msg='Retriever could not download "%s"' % self.url1)
  237. def testCrawlerFindsRegPageFromUrl(self):
  238. """Verify that the crawler is able to find a reg page from the given URL."""
  239. c = webforms_aggregator.Crawler(self.url1)
  240. self.assertTrue(
  241. c.Run(), msg='Crawler could not find the reg page of "%s"' % self.url1)
  242. def testCrawlerCannotFindNonExistentRegPageFromUrl(self):
  243. """Verify that the crawler won't find a non existent reg page
  244. from the given URL."""
  245. c = webforms_aggregator.Crawler(self.url2)
  246. self.assertFalse(
  247. c.Run(),
  248. msg='Crawler found a non existent reg page of "%s"' % self.url1)
  249. def testThreadedCrawlerFindsRegPageFromUrlsFile(self):
  250. """Verify the threaded crawler finds reg page from a file of URLs."""
  251. c = webforms_aggregator.ThreadedCrawler(self.files['url'])
  252. self.assertNotEqual(
  253. c.Run(), -1,
  254. msg='Threaded crawler could not find the reg page from the URLs file')
  255. if __name__ == '__main__':
  256. suite = unittest.TestLoader().loadTestsFromTestCase(
  257. WebformsAggregatorTest)
  258. unittest.TextTestRunner(verbosity=2).run(suite)