Examples of cascading.tuple.TupleEntryIterator

cascading.tuple.TupleEntryIterator
Class TupleEntryIterator provides an efficient Iterator for returning {@link TupleEntry} elements in anunderlying {@link Tuple} collection.

   * Returns a List containing all the tuple entries output from this flow.
   * Assumes flow has already run.
   * @return list of TupleEntries output from this flow.
   */
  public List<TupleEntry> getOutput(Flow flow) throws Exception {
    TupleEntryIterator output = flow.openSink();


    List<TupleEntry> result = new ArrayList<TupleEntry>();
    while (output.hasNext()) {
      TupleEntry e = output.next();
      // We have to make a copy of the TupleEntry here -- otherwise cascading
      // will overwrite its values somewhere deep in its innards.
      result.add(new TupleEntry(e));
    }
    return result;

View Full Code Here


    Flow flow = new LocalFlowConnector().connect( flowDef );


    flow.complete();


    TupleEntryIterator iterator = resultsTap.openForRead( flow.getFlowProcess() );


    while( iterator.hasNext() )
      System.out.println( iterator.next() );


    iterator.close();
    }

View Full Code Here


  private void verifySink( Flow flow, int expects ) throws IOException
    {
    int count = 0;


    TupleEntryIterator iterator = flow.openSink();


    while( iterator.hasNext() )
      {
      count++;
      System.out.println( "iterator.next() = " + iterator.next() );
      }


    iterator.close();


    assertEquals( "wrong number of values", expects, count );
    }

View Full Code Here

    return true;
    }


  public TupleEntryIterator openForRead( JobConf conf ) throws IOException
    {
    return new TupleEntryIterator( getSourceFields(), new TapIterator( this, conf ) );
    }

View Full Code Here

    for( int i = 0; i < 100; i++ )
      collector.add( new Tuple( "string", "" + i, i ) );


    collector.close();


    TupleEntryIterator iterator = tap.openForRead( getPlatform().getFlowProcess() );


    int count = 0;
    while( iterator.hasNext() )
      {
      iterator.next();
      count++;
      }


    iterator.close();


    assertEquals( "wrong size", 100, count );
    }

View Full Code Here

      }


    if( resultLength != -1 )
      validateLength( flow, resultLength );


    TupleEntryIterator iterator = flow.openSink();
    Object result = iterator.next().getObject( 1 );


    if( resultTuple != null )
      assertEquals( "not equal: ", resultTuple.toString(), result );
    else if( resultTuple == null )
      fail( "no result assertion made for:" + getName() + " with result: " + result );

View Full Code Here

    validateLength( flow, length, -1, regex, name );
    }


  public static void validateLength( Flow flow, int length, int size, Pattern regex, String name ) throws IOException
    {
    TupleEntryIterator iterator = name == null ? flow.openSink() : flow.openSink( name );
    validateLength( iterator, length, size, regex );
    }

View Full Code Here

    }


  public Fields parseFirstLine( FlowProcess flowProcess, Tap tap )
    {
    Fields sourceFields;
    TupleEntryIterator iterator = null;


    try
      {
      if( !tap.resourceExists( flowProcess.getConfigCopy() ) )
        throw new TapException( "unable to read fields from tap: " + tap + ", does not exist" );


      iterator = tap.openForRead( flowProcess );


      TupleEntry entry = iterator.hasNext() ? iterator.next() : null;


      if( entry == null )
        throw new TapException( "unable to read fields from tap: " + tap + ", is empty" );


      Object[] result = onlyParseLine( entry.getTuple().getString( 0 ) ); // don't coerce if type info is avail


      result = cleanParsedLine( result );


      Type[] inferred = inferTypes( result ); // infer type from field name, after removing quotes/escapes


      result = cleanFields( result ); // clean field names to remove any meta-data or manage case


      sourceFields = new Fields( Arrays.copyOf( result, result.length, Comparable[].class ) );


      if( inferred != null )
        sourceFields = sourceFields.applyTypes( inferred );
      }
    catch( IOException exception )
      {
      throw new TapException( "unable to read fields from tap: " + tap, exception );
      }
    finally
      {
      if( iterator != null )
        {
        try
          {
          iterator.close();
          }
        catch( IOException exception )
          {
          // do nothing
          }

View Full Code Here

    }


  private Throwable map( Object input )
    {
    Throwable localThrowable = null;
    TupleEntryIterator iterator = null;


    try
      {
      next.start( this );


      // input may be null
      iterator = source.openForRead( flowProcess, input );


      while( iterator.hasNext() )
        {
        TupleEntry tupleEntry;


        try
          {
          tupleEntry = iterator.next();
          flowProcess.increment( StepCounters.Tuples_Read, 1 );
          flowProcess.increment( SliceCounters.Tuples_Read, 1 );
          }
        catch( OutOfMemoryError error )
          {
          handleReThrowableException( "out of memory, try increasing task memory allocation", error );
          continue;
          }
        catch( CascadingException exception )
          {
          handleException( exception, null );
          continue;
          }
        catch( Throwable throwable )
          {
          handleException( new DuctException( "internal error", throwable ), null );
          continue;
          }


        next.receive( this, tupleEntry );
        }


      next.complete( this );
      }
    catch( Throwable throwable )
      {
      if( !( throwable instanceof OutOfMemoryError ) )
        LOG.error( "caught throwable", throwable );


      return throwable;
      }
    finally
      {
      try
        {
        if( iterator != null )
          iterator.close();
        }
      catch( Throwable currentThrowable )
        {
        if( !( currentThrowable instanceof OutOfMemoryError ) )
          LOG.warn( "failed closing iterator", currentThrowable );

View Full Code Here

      new Tuple( "c", 4 ),
      new Tuple( "d", 2 ),
      new Tuple( "e", 1 ),
    };


    TupleEntryIterator iterator = flow.openSink();
    int count = 0;


    while( iterator.hasNext() )
      assertEquals( results[ count++ ], iterator.next().getTuple() );


    iterator.close();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of cascading.tuple.TupleEntryIterator

backtype.hadoop.ElasticTap

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.DemoStatusTool

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

cascading.CascadingTestCase

cascading.dbmigrate.tap.DBMigrateTap

cascading.detail.PipeAssemblyTestBase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.