Examples of cascading.operation.Identity$Functor

cascading.operation.Identity
The Identity function simply passes incoming arguments back out again. Optionally argument fields can be renamed, and/or coerced into specific types.
During coercion, if the given type is a primitive ( {@code long}), and the tuple value is null, {@code 0} is returned.If the type is an Object ( {@code java.lang.Long}), and the tuple value is {@code null}, {@code null} is returned.


    int depth = 50;


    for( int i = 0; i < depth; i++ )
      {
      pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( 0 ) ), Fields.ALL );
      pipe = new Each( pipe, new Fields( "count" ), new Identity( new Fields( 0 ) ), Fields.ALL );
      pipe = new Each( pipe, new Fields( "line" ), new Identity(), Fields.REPLACE );
      pipe = new Each( pipe, new Fields( "count" ), new Identity(), Fields.REPLACE );
      pipe = new Each( pipe, new Fields( "line", "count" ), new Identity() );
      pipe = new Each( pipe, new Fields( "line", "count" ), new Identity( new Fields( "line2", "count2" ) ), new Fields( "line", "count2" ) );
      pipe = new Each( pipe, new Fields( "count2" ), new Identity( new Fields( "count" ) ), new Fields( "line", "count" ) );
      }


    int modulo = 1000000;


    pipe = new Each( pipe, new Fields( "line" ), new ExpressionFunction( new Fields( "hash" ), "line.hashCode() % " + modulo, String.class ), Fields.ALL ); // want some collisions


    pipe = new GroupBy( pipe, new Fields( "hash" ) );


    for( int i = 0; i < depth; i++ )
      pipe = new Every( pipe, new Fields( "count" ), new Sum( new Fields( "sum" + ( i + 1 ) ) ) );


    for( int i = 0; i < depth; i++ )
      {
      pipe = new Each( pipe, new Fields( "hash" ), new Identity( new Fields( 0 ) ), Fields.ALL );
      pipe = new Each( pipe, new Fields( "sum1" ), new Identity( new Fields( 0 ) ), Fields.ALL );
      pipe = new Each( pipe, new Fields( "hash", "sum1" ), new Identity(), Fields.SWAP );
      }


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();

View Full Code Here

    Pipe pipeLhs = new Each( new Pipe( "lhs" ), new Fields( "line" ), splitter );
    Pipe pipeRhs = new Each( new Pipe( "rhs" ), new Fields( "line" ), splitter );


    Pipe upperLower = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    upperLower = new Each( upperLower, new Identity() );


    Pipe lhsRhs = new HashJoin( pipeLhs, new Fields( "num" ), pipeRhs, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    lhsRhs = new Each( lhsRhs, new Identity() );


    Pipe merge = new Merge( "sink", Pipe.pipes( upperLower, lhsRhs ) );


    initialize( sources, sinks, merge );
    }

View Full Code Here


    Pipe splice1 = new CoGroup( pipeNum20, new Fields( "num" ), pipeNum101, new Fields( "num" ), new Fields( "num1", "num2" ) );


    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeNum102, new Fields( "num" ), new Fields( "num1", "num2", "num3" ) );


    splice2 = new Each( splice2, new Identity() );


    splice2 = new Pipe( "sink", splice2 );


    initialize( sources, sinks, splice2 );
    }

View Full Code Here

    Pipe pipeUpper1 = new Each( new Pipe( "upper1" ), new Fields( "line" ), function );
    Pipe pipeUpper2 = new Each( new Pipe( "upper2" ), new Fields( "line" ), function );


    Pipe splice1 = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );


    Pipe splice2 = new HashJoin( splice1, new Fields( "num1" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    Map<String, Tap> sources = createHashMap();

View Full Code Here

    Pipe pipeUpper1 = new Each( new Pipe( "upper1" ), new Fields( "line" ), splitter );
    Pipe pipeUpper2 = new Each( new Pipe( "upper2" ), new Fields( "line" ), splitter );


    Pipe splice1 = new HashJoin( pipeUpper1, new Fields( "num" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );


    Pipe splice2 = new HashJoin( "sink", splice1, new Fields( "num1" ), pipeLower, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    initialize( sources, sinks, splice2 );
    }

View Full Code Here


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new Each( pipe, new Fields( 1 ), new Identity(), new Fields( 2 ) ); // in:second out:all
    pipe = new Each( pipe, new Fields( 0 ), new Identity( new Fields( "_all" ) ), new Fields( 1 ) ); // in:all out:_all
    pipe = new GroupBy( pipe, new Fields( 0 ) ); // in:_all out:_all
    pipe = new Every( pipe, new Fields( 0 ), new Count(), new Fields( 0, 1 ) ); // in:_all out:_all,count


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps();

View Full Code Here

    Pipe pipeUpper1 = new Each( new Pipe( "upper1" ), new Fields( "line" ), function );
    Pipe pipeUpper2 = new Each( new Pipe( "upper2" ), new Fields( "line" ), function );


    Pipe splice1 = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );


    Pipe splice2 = new HashJoin( pipeUpper2, new Fields( "num1" ), splice1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    Map<String, Tap> sources = createHashMap();

View Full Code Here

    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe cogroup = new CoGroup( pipeA, new Fields( 1 ), pipeB, new Fields( 1 ) );


    cogroup = new Each( cogroup, new Identity() );


    sinks.put( cogroup.getName(), new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, cogroup ).getFlowSteps();

View Full Code Here

    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe splice = new CoGroup( pipeA, pipeB );


    splice = new Each( splice, new Identity() );


    sinks.put( splice.getName(), new Hfs( new TextLine(), "output/path" ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, splice ).getFlowSteps();

View Full Code Here


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new Each( pipe, new Identity() );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, pipe );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.operation.Identity$Functor

bixo.examples.crawl.DemoCrawlWorkflow

cascading.assembly.CrossTab

cascading.BasicPipesPlatformTest

cascading.cascade.CascadePlatformTest

cascading.cascade.hadoop.RiffleCascadePlatformTest

cascading.cascade.ParallelCascadePlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.