Examples of cascading.tuple.TupleEntry

cascading.tuple.TupleEntry
Class TupleEntry allows a {@link Tuple} instance and its declaring {@link Fields} instance to be used as a single object.
Once a TupleEntry is created, its Fields cannot be changed, but the Tuple instance it holds can be replaced or modified. The managed Tuple should not have elements added or removed, as this will break the relationship with the associated Fields instance.
If type information is provided on the Fields instance, all setters on this class will use that information to coerce the given object to the expected type.
For example, if position is is of type {@code long}, then {@code entry.setString(0, "9" )} will coerce the "9" to along {@code 9}. Thus, {@code entry.getObject(0) == 9l}.
No coercion is performed with the {@link #getObject(Comparable)} and {@link #getObject(int)} methods.
To set a value without coercion, see the {@link #setRaw(Comparable,Object)} and {@link #setRaw(int,Object)}methods. @see Fields @see Tuple

        TupleEntryIterator tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        
        int totalEntries = 0;
        boolean[] fetchedPages = new boolean[numPages];
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            FetchedDatum datum = new FetchedDatum(entry);
            String url = datum.getUrl();
            Assert.assertNotNull(url);
            
            // Verify that we got one of each page
            int idOffset = url.indexOf(".html") - 1;
            int pageId = Integer.parseInt(url.substring(idOffset, idOffset + 1));
            Assert.assertFalse(fetchedPages[pageId]);
            fetchedPages[pageId] = true;
        }
        
        Assert.assertEquals(numPages, totalEntries);
        tupleEntryIterator.close();
        
        validate = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath);
        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        totalEntries = 0;
        fetchedPages = new boolean[numPages];
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            StatusDatum sd = new StatusDatum(entry);
            Assert.assertEquals(UrlStatus.FETCHED, sd.getStatus());

View Full Code Here

        validate = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath);
        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        int totalEntries = 0;
        boolean[] fetchedPages = new boolean[numPages];
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            StatusDatum sd = new StatusDatum(entry);
            Assert.assertTrue(sd.getException() instanceof RedirectFetchException);

View Full Code Here

        
        Tap validate = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath);
        TupleEntryIterator tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        int totalEntries = 0;
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            StatusDatum sd = new StatusDatum(entry);
            Assert.assertEquals(UrlStatus.SKIPPED_INTERRUPTED, sd.getStatus());

View Full Code Here

        Tap validate = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), contentPath);
        TupleEntryIterator tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        
        int totalEntries = 0;
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;
            
            FetchedDatum datum = new FetchedDatum(entry);
            String payloadValue = (String)datum.getPayloadValue("key");
            Assert.assertNotNull(payloadValue);
            Assert.assertEquals("value", payloadValue);
        }
        
        Assert.assertEquals(1, totalEntries);
        tupleEntryIterator.close();
        
        validate = platform.makeTap(platform.makeBinaryScheme(StatusDatum.FIELDS), statusPath);
        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        totalEntries = 0;
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            totalEntries += 1;


            // Verify we can convert properly
            StatusDatum sd = new StatusDatum(entry);
            Assert.assertEquals(UrlStatus.FETCHED, sd.getStatus());

View Full Code Here

        flow.complete();
        
        Tap validate = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), contentPath);
        TupleEntryIterator tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        Assert.assertTrue(tupleEntryIterator.hasNext());
        TupleEntry te = tupleEntryIterator.next();
        String url = te.getString(FetchedDatum.URL_FN);
        Assert.assertTrue(url.contains("bixo-test-domain-1.com/page-1.html"));
        
        // Should only be one resulting page (for domain 1, page 1).
        Assert.assertFalse(tupleEntryIterator.hasNext());
    }

View Full Code Here

        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        
        int numEntries = 0;
        while (tupleEntryIterator.hasNext()) {
            numEntries += 1;
            TupleEntry entry = tupleEntryIterator.next();
            StatusDatum status = new StatusDatum(entry);
            Assert.assertEquals(UrlStatus.SKIPPED_TIME_LIMIT, status.getStatus());
        }
        
        Assert.assertEquals(10, numEntries);

View Full Code Here

        tupleEntryIterator = validate.openForRead(platform.makeFlowProcess());
        
        int numSkippedEntries = 0;
        int numFetchedEntries = 0;
        while (tupleEntryIterator.hasNext()) {
            TupleEntry entry = tupleEntryIterator.next();
            StatusDatum status = new StatusDatum(entry);
            if (status.getStatus() == UrlStatus.SKIPPED_PER_SERVER_LIMIT) {
                numSkippedEntries += 1;
            } else if (status.getStatus() == UrlStatus.FETCHED) {
                numFetchedEntries += 1;

View Full Code Here

        OperationCall<NullContext> oc = Mockito.mock(OperationCall.class);
        BufferCall<NullContext> bc = Mockito.mock(BufferCall.class);
        
        TupleEntryCollector collector = Mockito.mock(TupleEntryCollector.class);
        
        Mockito.when(bc.getGroup()).thenReturn(new TupleEntry(new Tuple("http://localhost:8089")));
        Mockito.when(bc.getArgumentsIterator()).thenReturn(getGroupedurlDatumList("http://localhost:8089").iterator());
        Mockito.when(bc.getOutputCollector()).thenReturn(collector);
        
        TestWebServer server = null;

View Full Code Here

        OperationCall<NullContext> oc = Mockito.mock(OperationCall.class);
        BufferCall<NullContext> bc = Mockito.mock(BufferCall.class);
        
        TupleEntryCollector collector = Mockito.mock(TupleEntryCollector.class);
        
        Mockito.when(bc.getGroup()).thenReturn(new TupleEntry(new Tuple("http://localhost:8089")));
        Mockito.when(bc.getArgumentsIterator()).thenReturn(getGroupedurlDatumList("http://localhost:8089").iterator());
        Mockito.when(bc.getOutputCollector()).thenReturn(collector);
        
        TestWebServer server = null;

View Full Code Here

                    // Re-add the thing from the top of the queue, since we're going to want to keep it around.
                    // This is safe to call with a null datum.
                    addToQueue(queueDatum);
                    
                    // Now get our next FetchSet from the Hadoop iterator.
                    FetchSetDatum iterDatum = new FetchSetDatum(new TupleEntry(_values.next()));
                    List<ScoredUrlDatum> urls = iterDatum.getUrls();
                    String ref = iterDatum.getGroupingRef();
                    
                    if (iterDatum.isSkipped()) {
                        trace("Skipping %d urls via iterator from %s (e.g. %s)", urls.size(), ref, urls.get(0).getUrl());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tuple.TupleEntry

bixo.examples.crawl.DemoStatusTool

bixo.fetcher.FetcherTest

bixo.operations.FetchBuffer$QueuedValues

bixo.operations.FilterAndScoreByUrlAndRobotsTest

bixo.pipes.AbstractFetchPipeTest

cascading.CascadingTestCase

cascading.DistanceUseCasePlatformTest

cascading.flow.stream.element.OperatorStage

cascading.function.FunctionPlatformTest

cascading.lingual.jdbc.JDBCPlatformTestCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.