Package com.crawl.control

Examples of com.crawl.control.Crawler

137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
     * This is the main function it is generating the output collection.
     */
    public String getDoAnalyze() {
        logger.info(this.toString());
        // Create the crawler object
        Crawler aCrawl = Crawler.getInstance();
        String craigslistURL = aCrawl.createUrl(
                this.getCategoryCode(),
                this.getLocationURL(),
                this.getKeyword());
        // Get all items
        crawlResult = aCrawl.crawlWebPages(craigslistURL, 2500);
        logger.debug("aResultColl Size=" + crawlResult.size());
        // Create analyzer object
        analyzer = new AnaCrack();
        // Do the analyzing and return the offers
        analyzerOffers = analyzer.analyse(
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
     */
    public static void main(String[] args) {
        StartMain.initLogger();

        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.SAN_FRANCISCO,
                "Apple");
       
        // 2. Step get all offers
        Collection<CrawlResultPackage> aResultColl =aCrawl.crawlWebPages(aUrl, 5000);

        logger.debug("aResultColl Size=" + aResultColl.size());

        // 3. Create analyzer object
        AnaCrack aAnaCrack = new AnaCrack();
View Full Code Here
21
22
23
24
25
26
27
28
  static Logger logger = Logger.getLogger(CrawlTest.class);

  @Test
  public void test_createCrawler() {
    logger.debug("Create Crawler class...");
    Crawler c = Crawler.getInstance();
    assertTrue(c != null);
  }
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    @Test
    public void test_callGetWebPagesMethod() {
        logger.debug("get 100 apple items");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);
       
        logger.debug("aCrawlResultColl.size()="+aCrawlResultColl.size());
       
        assertTrue(aCrawlResultColl.size()==100);
    } 
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    @Test
    public void test_MatchPattern() {
        logger.debug("Test MatchPattern...");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);

        String aString=aCrawl.getMatchPattern();
       
        logger.debug("MatchPatter="+aString);
       
        assertTrue(aString!=null);
    }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    @Test
    public void test_MatchPatternCompare() {
        logger.debug("Test MatchPattern... compare");

        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);

        String aString=aCrawl.getMatchPattern();
       
        logger.debug("MatchPatter="+aString);
       
        assertTrue(aString.compareTo(".*<a href=\"http://.*.craigslist.*/.*html.*>")==0);
                                     
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    @Test
    public void test_MatchPatternCompareLength() {
        logger.debug("Test MatchPattern... compare length");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);

        String aString=aCrawl.getMatchPattern();
       
        logger.info("MatchPatter="+aString+" length="+aString.length());
       
        assertTrue(aString.length()==43);
    }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    @Test
    public void test_url1() {
        logger.debug("Test url 1");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);
       
        logger.info("URL="+aUrl+" length="+aUrl.length());
       
        assertTrue(aUrl != null);
    }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    @Test
    public void test_url2() {
        logger.debug("Test url 2...");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);
       
        logger.info("URL="+aUrl+" length="+aUrl.length());
       
        assertTrue(aUrl.compareTo("http://sfbay.craigslist.org/search/sya/pen?query=Apple&maxAsk=100000&sort=pricedsc&srchType=A&s=") ==0);   
    }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
    @Test
    public void test_url3() {
        logger.debug("Test url 3...");
       
        // 1. Create the crawler object
        Crawler aCrawl = Crawler.getInstance();

        String aUrl=aCrawl.createUrl(
                CraigslistCategoryEnum.FOR_SALE__COMPUTER,
                CraigslistAreasEnum.PENINSULA,
                "Apple");
              
        Collection<CrawlResultPackage> aCrawlResultColl=aCrawl.crawlWebPages(aUrl, 100);
               
        logger.debug("URL="+aUrl+" length="+aUrl.length());
       
        assertTrue(aUrl.length()==96);   
    }
View Full Code Here
TOP

Related Classes of com.crawl.control.Crawler

  • com.craigslist.starter.StartMain
  • com.crawl.model.CrawlResultPackage
  • com.test.CrawlTest
  • com.ui.AnalyzerBean
  • org.jsoup.nodes.Document
  • org.jsoup.select.Elements
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.