Examples of com.scaleunlimited.cascading.Payload

com.scaleunlimited.cascading.Payload

public class FetchSetDatumTest {


    private static List<ScoredUrlDatum> makeUrls(int count, String payloadFieldName) {
        List<ScoredUrlDatum> results = new LinkedList<ScoredUrlDatum>();


       Payload payload = new Payload();


        for (int i = 0; i < count; i++) {
            ScoredUrlDatum url = new ScoredUrlDatum("http://domain.com/page-" + i, "key", UrlStatus.UNFETCHED, 1.0);
            if (payloadFieldName != null) {
                payload.put(payloadFieldName, "value-" + i);
                url.setPayload(payload);
            }


            results.add(url);
        }

View Full Code Here

    }
    
    @Test
    public void testCreatingFromScoredUrlDatum() throws Exception {
        ScoredUrlDatum sud = new ScoredUrlDatum("url", "groupKey", UrlStatus.UNFETCHED);
        Payload payload = new Payload();
        payload.put("key", "value");
        sud.setPayload(payload);
        
        FetchedDatum fd = new FetchedDatum(sud);
        assertEquals("value", fd.getPayload().get("key"));
    }

View Full Code Here


public class StatusDatumTest {


    @Test
    public void testConstructorWithPayload() throws Exception {
        Payload payload = new Payload();
        payload.put("key", "value");
        StatusDatum sd = new StatusDatum("url", UrlStatus.UNFETCHED, payload);
        
        assertEquals("value", sd.getPayload().get("key"));
    }

View Full Code Here

    protected void testFetchPipe(BixoPlatform platform) throws Exception {
        // System.setProperty("bixo.root.level", "TRACE");
        final int numPages = 10;
        final int port = 8089;
        
        Tap in = makeInputData(platform, "testFetchPipe", "localhost:" + port, numPages, new Payload());


        Pipe pipe = new Pipe("urlSource");
        BaseScoreGenerator scorer = new FixedScoreGenerator();
        BaseFetcher fetcher = new SimpleHttpFetcher(ConfigUtils.BIXO_TEST_AGENT);
        FetchPipe fetchPipe = new FetchPipe(pipe, scorer, fetcher, 1);

View Full Code Here

        // System.setProperty("bixo.root.level", "TRACE");
        
        final int numPages = 1;
        final int port = 8089;
        
        Payload payload = new Payload();
        payload.put("payload-field-1", 1);
        Tap in = makeInputData(platform, "testRedirectException", "localhost:" + port, numPages, payload);


        Pipe pipe = new Pipe("urlSource");
        BaseScoreGenerator scorer = new FixedScoreGenerator();
        FetcherPolicy policy = new FetcherPolicy();
        policy.setRedirectMode(RedirectMode.FOLLOW_TEMP);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        FetchPipe fetchPipe = new FetchPipe(pipe, scorer, fetcher, 1);
        
        String output = "build/test/FetchPipeTest/testRedirectException";
        BasePath outputPath = platform.makePath(output);
        BasePath statusPath = platform.makePath(outputPath, "status");
        BasePath contentPath = platform.makePath(outputPath, "content");
        Tap status = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath, SinkMode.REPLACE);
        Tap content = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), contentPath, SinkMode.REPLACE);


        // Finally we can run it.
        FlowConnector flowConnector = platform.makeFlowConnector();
        Flow flow = flowConnector.connect(in, FetchPipe.makeSinkMap(status, content), fetchPipe);
        TestWebServer webServer = null;
        
        try {
            webServer = new TestWebServer(new RedirectResponseHandler(true), port);
            flow.complete();
        } finally {
            webServer.stop();
        }
        
        // Verify numPages fetched and numPages status entries were saved.
        Tap validate = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), contentPath);
        TupleEntryIterator tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        Assert.assertFalse(tupleEntryIterator.hasNext());
        tupleEntryIterator.close();
        
        validate = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath);
        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        int totalEntries = 0;
        boolean[] fetchedPages = new boolean[numPages];
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            StatusDatum sd = new StatusDatum(entry);
            Assert.assertTrue(sd.getException() instanceof RedirectFetchException);
            RedirectFetchException redirectException =
                (RedirectFetchException)(sd.getException());
            Assert.assertEquals(RedirectResponseHandler.REDIRECT_TARGET_URL,
                                redirectException.getRedirectedUrl());
            Assert.assertEquals(payload.get("payload-field-1"),
                                sd.getPayloadValue("payload-field-1"));
            
            // Verify that we got one of each page
            String url = sd.getUrl();
            Assert.assertNotNull(url);

View Full Code Here

        // TODO CSc - re-enable this test, when termination really works.
        // Assert.assertEquals(numPages, totalEntries);
    }
    
    protected void testPayloads(BixoPlatform platform) throws Exception {
        Payload payload = new Payload();
        payload.put("key", "value");
        Tap in = makeInputData(platform, "testPayloads", 1, 1, payload);


        Pipe pipe = new Pipe("urlSource");
        BaseFetcher fetcher = new FakeHttpFetcher(false, 10);
        BaseScoreGenerator scorer = new FixedScoreGenerator();

View Full Code Here

            throw e;
        }
    }


    public FetchedResult fetch(String url) throws BaseFetchException{
      return fetch(new HttpGet(), url, new Payload());
    }

View Full Code Here




    @Override
    public FetchedDatum get(ScoredUrlDatum datum) throws BaseFetchException {
        String url = datum.getUrl();
        Payload payload = datum.getPayload();
        logPayload(url, payload);
        
        // Create a simple HTML page here, where we fill in the URL as
        // the field, and return that as the BytesWritable. we could add
        // more of the datum values to the template if we cared.

View Full Code Here

TOP

Related Classes of com.scaleunlimited.cascading.Payload

bixo.datum.FetchedDatumTest

bixo.datum.FetchSetDatumTest

bixo.datum.StatusDatumTest

bixo.fetcher.LoggingFetcher

bixo.fetcher.SimpleHttpFetcher

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.