123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298 |
- #!/usr/bin/env python
- # Copyright (c) 2011 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- import logging
- import os
- import subprocess
- import tempfile
- import unittest
- # Same name as the aggregator module name.
- import webforms_aggregator
- logger = logging.getLogger(webforms_aggregator.__name__)
- console = logging.StreamHandler()
- logger.addHandler(console)
- # Commenting out the following line will set logger level to default: WARNING
- logger.setLevel(logging.INFO)
- class WebformsAggregatorTest(unittest.TestCase):
- """Unit tests for the webforms_aggregator module."""
- PORT1 = 8002
- PORT2 = 8003
- HOME_CONTENT = """
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
- "http://www.w3.org/TR/html4/loose.dtd">
- <html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>%s</title>
- </head>
- <body>
- <h1>%s</h1>
- <p>This is a mock site. Its mere purpose is to contribute towards testing \
- the aggregator crawler.</p>
- <ul>
- <li><a href="%s">page1</a></li>
- <li><a href="%s">page2</a></li>
- <li><a href="%s">page3</a></li>
- </ul>
- <hr>
- <p>
- <a href="%s">sign in</a>
- </p>
- </body>
- </html>
- """
- SIMPLE_PAGE_CONTENT = """
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
- "http://www.w3.org/TR/html4/loose.dtd">
- <html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>%s</title>
- </head>
- <body>
- <h1>%s</h1>
- <p>%s</p>
- <ul>
- <li><a href="%s">%s</a></li>
- <li><a href="%s">%s</a></li>
- </ul>
- <hr>
- <p>
- <a href="%s">return to home page</a>
- </p>
- </body>
- </html>
- """
- SIGNIN_CONTENT = """
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
- "http://www.w3.org/TR/html4/loose.dtd">
- <html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>%s</title>
- </head>
- <body>
- <h1>Sign in!</h1>
- <h3>%s</h3>
- <form>
- <label>User name: </label><input type="text"><br><br>
- <label>password: </label><input type="password"><br><br>
- <input type="submit" value="Sign in">
- </form>
- <hr>
- <p><a href="%s">return to home page</a></p>
- </body>
- </html>
- """
- REG_CONTENT = """
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" \
- "http://www.w3.org/TR/html4/loose.dtd">
- <html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>%s</title>
- </head>
- <body>
- <h1>Create a user account!</h1>
- <h3>Enter your data below:</h3>
- <form method="get">
- <label>First name: </label><input type="text"><br><br>
- <label>Surname: </label><input type="text"><br><br>
- <label>User name: </label><input type="text"><br><br>
- <label>password: </label><input type="password"><br><br>
- <label>retype password: </label><input type="password"><br><br>
- <input type="submit" value="Register">
- </form>
- <hr>
- <p><a href="%s">return to home page</a></p>
- </body>
- </html>
- """
- def CreateMockSiteOne(self):
- """Site One has a registration form.
- """
- self.files['site1_home'] = 'site1_index.html'
- self.files['site1_page1'] = 'site1_page1.html'
- self.files['site1_page2'] = 'site1_page2.html'
- self.files['site1_page3'] = 'site1_page3.html'
- self.files['site1_signin'] = 'site1_signin.html'
- self.files['site1_reg'] = 'site1_register.html'
- file_content = {}
- file_content[self.files['site1_home']] = self.HOME_CONTENT % (
- 'Site One home page', 'Welcome to site one. It has a reg page!',
- self.files['site1_page1'], self.files['site1_page2'],
- self.files['site1_page3'], self.files['site1_signin'])
- file_content[self.files['site1_page1']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site One page 1',
- 'Page 1!', 'This is a useless page. It does almost nothing.',
- self.files['site1_page2'], 'page 2', self.files['site1_page3'],
- 'page 3', self.files['site1_home'])
- file_content[self.files['site1_page2']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site One page 2', 'Page 2!',
- 'This is another useless page. It does almost what the page 1 does.',
- self.files['site1_page1'], 'page 1', self.files['site1_page3'],
- 'page 3', self.files['site1_home'])
- file_content[self.files['site1_page3']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site One page 3', 'Page 3!',
- "This is the last useless page. It doesn't do anything useful at all.",
- self.files['site1_page1'], 'page 1', self.files['site1_page2'],
- 'page 2', self.files['site1_home'])
- file_content[self.files['site1_signin']] = self.SIGNIN_CONTENT % (
- 'Site One signin',
- 'If you don\'t have a user account click <a href="%s">here</a>.' \
- % self.files['site1_reg'],
- self.files['site1_home'])
- file_content[self.files['site1_reg']] = self.REG_CONTENT % (
- 'Site One signin', self.files['site1_home'])
- for filename, content in file_content.iteritems():
- f = open(filename, 'w')
- try:
- f.write(content)
- finally:
- f.close()
- def CreateMockSiteTwo(self):
- """ Site Two has no registration page."""
- self.files['site2_home'] = 'site2_index.html'
- self.files['site2_page1'] = 'site2_page1.html'
- self.files['site2_page2'] = 'site2_page2.html'
- self.files['site2_page3'] = 'site2_page3.html'
- self.files['site2_signin'] = 'site2_signin.html'
- file_content = {}
- file_content[self.files['site2_home']] = self.HOME_CONTENT % (
- 'Site Two home page', 'Welcome to site two. It has no reg page!',
- self.files['site2_page1'], self.files['site2_page2'],
- self.files['site2_page3'], self.files['site2_signin'])
- file_content[self.files['site2_page1']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site Two page 1',
- 'Page 1!', 'This is a useless page. It does almost nothing.',
- self.files['site2_page2'], 'page 2', self.files['site2_page3'],
- 'page 3', self.files['site2_home'])
- file_content[self.files['site2_page2']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site Two page 2', 'Page 2!',
- 'This is another useless page. It does almost what the page 1 does.',
- self.files['site2_page1'], 'page 1', self.files['site2_page3'],
- 'page 3', self.files['site2_home'])
- file_content[self.files['site2_page3']] = self.SIMPLE_PAGE_CONTENT % (
- 'Site Two page 3', 'Page 3!',
- "This is the last useless page. It doesn't do anything useful at all.",
- self.files['site2_page1'], 'page 1', self.files['site2_page2'],
- 'page 2', self.files['site2_home'])
- file_content[self.files['site2_signin']] = self.SIGNIN_CONTENT % (
- 'Site Two signin', 'You cannot register online with this site.',
- self.files['site2_home'])
- for filename, content in file_content.iteritems():
- f = open(filename, 'w')
- try:
- f.write(content)
- finally:
- f.close()
- def setUp(self):
- self.cwd = os.getcwdu()
- self.temp_dir = tempfile.mkdtemp()
- os.chdir(self.temp_dir)
- self.files = {}
- self.CreateMockSiteOne()
- self.CreateMockSiteTwo()
- self.files['cookie'] = 'test.cookie'
- self.url1 = 'http://localhost:%s/%s' % (self.PORT1,
- self.files['site1_home'])
- self.url2 = 'http://localhost:%s/%s' % (self.PORT2,
- self.files['site2_home'])
- self.domain1 = 'localhost:%s' %self.PORT1
- self.files['url'] = 'urls.txt'
- url_file_handler = open(self.files['url'], 'w')
- try:
- url_file_handler.write('URLs to crawl:')
- url_file_handler.write(os.linesep)
- for url in (self.url1, self.url2):
- url_file_handler.write(url)
- url_file_handler.write(os.linesep)
- finally:
- url_file_handler.close()
- command_line = 'python -u -m SimpleHTTPServer %s' % self.PORT1
- args = command_line.split()
- self.server1 = subprocess.Popen(
- args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- self.server1.stdout.readline() # Needed in order for the server to start up
- command_line = 'python -u -m SimpleHTTPServer %s' % self.PORT2
- args = command_line.split()
- self.server2 = subprocess.Popen(
- args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- self.server2.stdout.readline() # Needed in order for the server to start up
- def tearDown(self):
- self.server1.terminate()
- self.server2.terminate()
- for filename in self.files.values():
- if os.path.isfile(filename):
- os.unlink(filename)
- os.chdir(self.cwd)
- os.rmdir(self.temp_dir)
- def testRetrieverDownloadsPage(self):
- """Verify the retriever can download a page."""
- r = webforms_aggregator.Retriever(self.url1, self.domain1,
- self.files['cookie'])
- self.assertTrue(r.Download(),
- msg='Retriever could not download "%s"' % self.url1)
- def testCrawlerFindsRegPageFromUrl(self):
- """Verify that the crawler is able to find a reg page from the given URL."""
- c = webforms_aggregator.Crawler(self.url1)
- self.assertTrue(
- c.Run(), msg='Crawler could not find the reg page of "%s"' % self.url1)
- def testCrawlerCannotFindNonExistentRegPageFromUrl(self):
- """Verify that the crawler won't find a non existent reg page
- from the given URL."""
- c = webforms_aggregator.Crawler(self.url2)
- self.assertFalse(
- c.Run(),
- msg='Crawler found a non existent reg page of "%s"' % self.url1)
- def testThreadedCrawlerFindsRegPageFromUrlsFile(self):
- """Verify the threaded crawler finds reg page from a file of URLs."""
- c = webforms_aggregator.ThreadedCrawler(self.files['url'])
- self.assertNotEqual(
- c.Run(), -1,
- msg='Threaded crawler could not find the reg page from the URLs file')
- if __name__ == '__main__':
- suite = unittest.TestLoader().loadTestsFromTestCase(
- WebformsAggregatorTest)
- unittest.TextTestRunner(verbosity=2).run(suite)
|