Examples of cascading.operation.Identity

cascading.operation.Identity
The Identity function simply passes incoming arguments back out again. Optionally argument fields can be renamed, and/or coerced into specific types.
During coercion, if the given type is a primitive ( {@code long}), and the tuple value is null, {@code 0} is returned.If the type is an Object ( {@code java.lang.Long}), and the tuple value is {@code null}, {@code null} is returned.

    Pipe pipeUpper1 = new Each( "upper1", new Fields( "line" ), splitter );
    Pipe pipeUpper2 = new Each( "upper2", new Fields( "line" ), splitter );


    Pipe splice1 = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );


    splice1 = new GroupBy( splice1, new Fields( 0 ) );


    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    splice2 = new Each( splice2, new Identity() );


    splice2 = new GroupBy( splice2, new Fields( 0 ) );


    splice2 = new CoGroup( splice2, new Fields( "num1" ), splice1, new Fields( "num1" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3", "num4", "char4", "num5", "char5" ) );

View Full Code Here

    sinks.put( "output2", sink2 );


    Pipe pipeLower1 = new Pipe( "lower1" );
    Pipe pipeUpper1 = new Pipe( "upper1" );


    Pipe pipeLower2 = new Each( pipeLower1, new Identity() );
    pipeLower2 = new GroupBy( pipeLower2, new Fields( "num", "char" ) );
    pipeLower2 = new Every( pipeLower2, new Fields( "num", "char" ), new Count(), new Fields( "num", "char" ) );


    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Pipe( "lower2", pipeLower1 );


    pipeUpper1 = new Each( pipeUpper1, new Identity() );
    pipeUpper1 = new Each( pipeUpper1, new Identity() );
    pipeUpper1 = new Each( pipeUpper1, new Identity() );


    Pipe splice1 = new CoGroup( "group", Pipe.pipes( pipeLower1, pipeLower2, pipeUpper1 ), Fields.fields( new Fields( "num" ), new Fields( "num" ), new Fields( "num" ) ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ), new InnerJoin() );


    Pipe output1 = new Each( splice1, AssertionLevel.VALID, new AssertNotNull() );
    output1 = new Each( output1, new Identity() );
    output1 = new Pipe( "output1", output1 );


    Pipe output2 = new Each( splice1, AssertionLevel.VALID, new AssertNull() );
    output2 = new Each( output2, new Identity() );
    output2 = new Pipe( "output2", output2 );


    Flow flow = null;
    try
      {

View Full Code Here

    sinks.put( "output2", sink2 );


    Pipe pipeLower1 = new Pipe( "lower1" );


    Pipe pipe = new GroupBy( pipeLower1, new Fields( 0 ) );
    Pipe left = new Each( new Pipe( "output1", pipe ), new Identity() );
    Pipe right = new Each( new Pipe( "output2", pipe ), new Identity() );


    Flow flow = null;
    try
      {
      flow = getPlatform().getFlowConnector().connect( sources, sinks, Pipe.pipes( left, right ) );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    Function parser = new RegexParser( new Fields( 0 ), "^[^ ]*" );
    pipe = new Each( pipe, new Fields( "line" ), parser, Fields.REPLACE );
    pipe = new Each( pipe, new Fields( "line" ), new Identity( Fields.ARGS ), Fields.REPLACE );
    pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "line2" ) ), Fields.REPLACE );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );
      fail( "did not fail" );

View Full Code Here

    Pipe pipeUpper1 = new Each( new Pipe( "upper1" ), new Fields( "line" ), function );
    Pipe pipeUpper2 = new Each( new Pipe( "upper2" ), new Fields( "line" ), function );


    Pipe splice1 = new HashJoin( pipeUpper1, new Fields( "num" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );


    Pipe splice2 = new HashJoin( pipeLower, new Fields( "num" ), splice1, new Fields( "num1" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    Map<String, Tap> sources = createHashMap();

View Full Code Here

    Pipe pipeUpper1 = new Each( "upper1", new Fields( "line" ), function );
    Pipe pipeUpper2 = new Each( "upper2", new Fields( "line" ), function );


    Pipe splice1 = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    splice1 = new Each( splice1, new Identity() );
    splice1 = new Each( splice1, new Identity() );
    splice1 = new Each( new Pipe( "last", splice1 ), new Identity() );


    splice1 = new GroupBy( splice1, new Fields( 0 ) );


    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    if( testBetween )
      splice2 = new Each( new Pipe( "before", splice2 ), new Identity() );


    splice2 = new Each( new Pipe( "remove", splice2 ), new Identity() );


    if( testBetween )
      splice2 = new Each( new Pipe( "after", splice2 ), new Identity() );


    splice2 = new GroupBy( splice2, new Fields( 0 ) );


    Pipe tail = new CoGroup( splice2, new Fields( "num1" ), splice1, new Fields( "num1" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3", "num4", "char4", "num5", "char5" ) );


    tail = new Each( tail, new Identity() );


    Pipe lhsTail = new Pipe( "lhsTail", tail );


    lhsTail = new Each( lhsTail, new Identity() );


    Pipe rhsTail = new Pipe( "rhsTail", tail );


    rhsTail = new Each( rhsTail, new Identity() );


    Map<String, Tap> sources = createHashMap();


    sources.put( "lower", new NonTap( new Fields( "offset", "line" ) ) );
    sources.put( "upper1", new NonTap( new Fields( "offset", "line" ) ) );

View Full Code Here

    Pipe apachePipe = new Pipe( "apache" );


    apachePipe = new Each( apachePipe, new Fields( "line" ), apacheCommonParser );
    apachePipe = new Each( apachePipe, new Insert( new Fields( "col" ), 1 ), Fields.ALL );
    apachePipe = new Each( apachePipe, new Fields( "ip" ), new RegexParser( new Fields( "octet" ), "^[^.]*" ), new Fields( "col", "status", "event", "octet", "size" ) );
    apachePipe = new Each( apachePipe, new Fields( "octet" ), new Identity( long.class ), Fields.REPLACE );


    Fields groupApache = new Fields( "octet" );
    groupApache.setComparator( "octet", getPlatform().getLongComparator( reverseSort ) );


    Pipe ipPipe = new Pipe( "ip" );


    ipPipe = new Each( ipPipe, new Fields( "line" ), new Identity( new Fields( "rawip" ) ) );
    ipPipe = new Each( ipPipe, new Fields( "rawip" ), new RegexParser( new Fields( "rawoctet" ), "^[^.]*" ), new Fields( "rawoctet" ) );
    ipPipe = new Each( ipPipe, new Fields( "rawoctet" ), new Identity( long.class ), Fields.REPLACE );


    Fields groupIP = new Fields( "rawoctet" );
    groupIP.setComparator( "rawoctet", getPlatform().getLongComparator( reverseSort ) );


    Pipe pipe = new CoGroup( apachePipe, groupApache, ipPipe, groupIP );


    pipe = new Each( pipe, new Identity() ); // let's force the stack to be exercised


    Map<Object, Object> properties = getProperties();


    if( getPlatform().isMapReduce() && getPlatform().getNumMapTasks( properties ) != null )
      getPlatform().setNumMapTasks( properties, 13 );

View Full Code Here


    Pipe pipe = new Pipe( "comprehensivesort" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ), "\\s" ) );


    pipe = new Each( pipe, new Fields( "num" ), new Identity( long.class ), Fields.REPLACE );


    Fields groupFields = new Fields( "num" );


    if( testCase[ 0 ] )
      groupFields.setComparator( "num", useCollectionsComparator ? Collections.reverseOrder() : getPlatform().getLongComparator( true ) );

View Full Code Here

    // DateParser.APACHE declares: "ts"
    pipe = new Each( pipe, new Fields( "time" ), new DateParser( "dd/MMM/yyyy:HH:mm:ss Z" ), new Fields( "col", "status", "ts", "event", "ip", "size" ) );


    pipe = new GroupBy( pipe, new Fields( "col" ), new Fields( "does-not-exist" ) );


    pipe = new Each( pipe, new Identity() ); // let's force the stack to be exercised


    try
      {
      getPlatform().getFlowConnector().connect( source, sink, pipe );
      fail( "did not throw exception" );

View Full Code Here


    Map<Object, Object> props = new HashMap<Object, Object>();
    TupleEntrySchemeIteratorProps.setPermittedExceptions( props, java.io.EOFException.class );


    Pipe pipe = new Pipe( "data" );
    pipe = new Each( pipe, new Identity() );


    FlowDef flowDef = FlowDef.flowDef().addSource( pipe, source ).addTailSink( pipe, sink );
    Flow flow = getPlatform().getFlowConnector( props ).connect( flowDef );
    flow.complete();
    validateLength( flow.openSink(), 307 );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.operation.Identity

bixo.examples.crawl.DemoCrawlWorkflow

cascading.assembly.CrossTab

cascading.BasicPipesPlatformTest

cascading.cascade.CascadePlatformTest

cascading.cascade.hadoop.RiffleCascadePlatformTest

cascading.cascade.ParallelCascadePlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.