Package cascading.tuple

Examples of cascading.tuple.TupleEntryIterator


   * Returns a List containing all the tuple entries output from this flow.
   * Assumes flow has already run.
   * @return list of TupleEntries output from this flow.
   */
  public List<TupleEntry> getOutput(Flow flow) throws Exception {
    TupleEntryIterator output = flow.openSink();

    List<TupleEntry> result = new ArrayList<TupleEntry>();
    while (output.hasNext()) {
      TupleEntry e = output.next();
      // We have to make a copy of the TupleEntry here -- otherwise cascading
      // will overwrite its values somewhere deep in its innards.
      result.add(new TupleEntry(e));
    }
    return result;
View Full Code Here


    Flow flow = new LocalFlowConnector().connect( flowDef );

    flow.complete();

    TupleEntryIterator iterator = resultsTap.openForRead( flow.getFlowProcess() );

    while( iterator.hasNext() )
      System.out.println( iterator.next() );

    iterator.close();
    }
View Full Code Here

  private void verifySink( Flow flow, int expects ) throws IOException
    {
    int count = 0;

    TupleEntryIterator iterator = flow.openSink();

    while( iterator.hasNext() )
      {
      count++;
      System.out.println( "iterator.next() = " + iterator.next() );
      }

    iterator.close();

    assertEquals( "wrong number of values", expects, count );
    }
View Full Code Here

    return true;
    }

  public TupleEntryIterator openForRead( JobConf conf ) throws IOException
    {
    return new TupleEntryIterator( getSourceFields(), new TapIterator( this, conf ) );
    }
View Full Code Here

    for( int i = 0; i < 100; i++ )
      collector.add( new Tuple( "string", "" + i, i ) );

    collector.close();

    TupleEntryIterator iterator = tap.openForRead( getPlatform().getFlowProcess() );

    int count = 0;
    while( iterator.hasNext() )
      {
      iterator.next();
      count++;
      }

    iterator.close();

    assertEquals( "wrong size", 100, count );
    }
View Full Code Here

      }

    if( resultLength != -1 )
      validateLength( flow, resultLength );

    TupleEntryIterator iterator = flow.openSink();
    Object result = iterator.next().getObject( 1 );

    if( resultTuple != null )
      assertEquals( "not equal: ", resultTuple.toString(), result );
    else if( resultTuple == null )
      fail( "no result assertion made for:" + getName() + " with result: " + result );
View Full Code Here

    validateLength( flow, length, -1, regex, name );
    }

  public static void validateLength( Flow flow, int length, int size, Pattern regex, String name ) throws IOException
    {
    TupleEntryIterator iterator = name == null ? flow.openSink() : flow.openSink( name );
    validateLength( iterator, length, size, regex );
    }
View Full Code Here

    }

  public Fields parseFirstLine( FlowProcess flowProcess, Tap tap )
    {
    Fields sourceFields;
    TupleEntryIterator iterator = null;

    try
      {
      if( !tap.resourceExists( flowProcess.getConfigCopy() ) )
        throw new TapException( "unable to read fields from tap: " + tap + ", does not exist" );

      iterator = tap.openForRead( flowProcess );

      TupleEntry entry = iterator.hasNext() ? iterator.next() : null;

      if( entry == null )
        throw new TapException( "unable to read fields from tap: " + tap + ", is empty" );

      Object[] result = onlyParseLine( entry.getTuple().getString( 0 ) ); // don't coerce if type info is avail

      result = cleanParsedLine( result );

      Type[] inferred = inferTypes( result ); // infer type from field name, after removing quotes/escapes

      result = cleanFields( result ); // clean field names to remove any meta-data or manage case

      sourceFields = new Fields( Arrays.copyOf( result, result.length, Comparable[].class ) );

      if( inferred != null )
        sourceFields = sourceFields.applyTypes( inferred );
      }
    catch( IOException exception )
      {
      throw new TapException( "unable to read fields from tap: " + tap, exception );
      }
    finally
      {
      if( iterator != null )
        {
        try
          {
          iterator.close();
          }
        catch( IOException exception )
          {
          // do nothing
          }
View Full Code Here

    }

  private Throwable map( Object input )
    {
    Throwable localThrowable = null;
    TupleEntryIterator iterator = null;

    try
      {
      next.start( this );

      // input may be null
      iterator = source.openForRead( flowProcess, input );

      while( iterator.hasNext() )
        {
        TupleEntry tupleEntry;

        try
          {
          tupleEntry = iterator.next();
          flowProcess.increment( StepCounters.Tuples_Read, 1 );
          flowProcess.increment( SliceCounters.Tuples_Read, 1 );
          }
        catch( OutOfMemoryError error )
          {
          handleReThrowableException( "out of memory, try increasing task memory allocation", error );
          continue;
          }
        catch( CascadingException exception )
          {
          handleException( exception, null );
          continue;
          }
        catch( Throwable throwable )
          {
          handleException( new DuctException( "internal error", throwable ), null );
          continue;
          }

        next.receive( this, tupleEntry );
        }

      next.complete( this );
      }
    catch( Throwable throwable )
      {
      if( !( throwable instanceof OutOfMemoryError ) )
        LOG.error( "caught throwable", throwable );

      return throwable;
      }
    finally
      {
      try
        {
        if( iterator != null )
          iterator.close();
        }
      catch( Throwable currentThrowable )
        {
        if( !( currentThrowable instanceof OutOfMemoryError ) )
          LOG.warn( "failed closing iterator", currentThrowable );
View Full Code Here

      new Tuple( "c", 4 ),
      new Tuple( "d", 2 ),
      new Tuple( "e", 1 ),
    };

    TupleEntryIterator iterator = flow.openSink();
    int count = 0;

    while( iterator.hasNext() )
      assertEquals( results[ count++ ], iterator.next().getTuple() );

    iterator.close();
    }
View Full Code Here

TOP

Related Classes of cascading.tuple.TupleEntryIterator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.