1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.any23.plugin.crawler;
19
20 import edu.uci.ics.crawler4j.crawler.Page;
21 import org.apache.any23.Any23OnlineTestBase;
22 import org.junit.Assert;
23 import org.junit.Test;
24 import org.slf4j.Logger;
25 import org.slf4j.LoggerFactory;
26
27 import java.io.File;
28 import java.net.URL;
29 import java.util.HashSet;
30 import java.util.Set;
31 import java.util.Iterator;
32
33
34
35
36
37
38 public class SiteCrawlerTest extends Any23OnlineTestBase {
39
40 public static final Logger logger = LoggerFactory.getLogger(SiteCrawlerTest.class);
41
42
43
44
45
46
47 @Test
48 public void testSiteCrawling() throws Exception {
49 assumeOnlineAllowed();
50
51 File tmpFile = File.createTempFile("site-crawler-test", ".storage");
52 tmpFile.delete();
53
54 final SiteCrawler controller = new SiteCrawler(tmpFile);
55 controller.setMaxPages(100);
56 logger.info("Crawler4j: Setting max num of pages to: " + controller.getMaxPages());
57 controller.setPolitenessDelay(500);
58 logger.info("Crawler4j: Setting Politeness delay to: " + controller.getPolitenessDelay() + "ms");
59
60 final Set<String> distinctPages = new HashSet<String>();
61 controller.addListener(new CrawlerListener() {
62 @Override
63 public void visitedPage(Page page) {
64 distinctPages.add( page.getWebURL().getURL() );
65 Iterator<String> it = distinctPages.iterator();
66 while (it.hasNext()) {
67 logger.info("Crawler4j: Fetching page - " + it.next());
68 }
69 }
70 });
71
72 controller.start( new URL("http://any23.apache.org/"), false);
73
74 synchronized (this) {
75 this.wait(15 * 1000);
76 }
77 controller.stop();
78
79 logger.info("Distinct pages: " + distinctPages.size());
80 Assert.assertTrue("Expected some page crawled.", distinctPages.size() > 0);
81 }
82
83 }