Package cascading.operation

Examples of cascading.operation.Identity

73
74
75
76
77
78
79
80
81
82
83
84
        // TODO: Make secondary sorting optional, and come up with a function to generate
        // a sortable key (vs just using the same serialization as for sharding).
        out = new Each(out, new Fields(0), new MakeSortableKey(keySortField, spec), Fields.ALL);

        //put in order of shard, key, value, sortablekey
        out = new Each(out, new Fields(2, 0, 1, 3), new Identity(), Fields.RESULTS);
        out = new GroupBy(out, new Fields(0), new Fields(3)); // group by shard

        // emit shard, key, value
        out = new Each(out, new Fields(0, 1, 2), new Identity());
        setTails(out);
    }
View Full Code Here
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
    Pipe pipeUpper1 = new Each( new Pipe( "upper1" ), new Fields( "line" ), splitter );
    Pipe pipeUpper2 = new Each( new Pipe( "upper2" ), new Fields( "line" ), splitter );

    Pipe splice1 = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper1, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );

    splice1 = new Each( splice1, new Identity() );

    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );

    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice2 );
View Full Code Here
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
    Pipe splice1 = new CoGroup( pipeNum20, new Fields( "num" ), pipeNum101, new Fields( "num" ), new Fields( "num1", "num2" ) );

    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeNum102, new Fields( "num" ), new Fields( "num1", "num2", "num3" ) );

    splice2 = new Each( splice2, new Identity() );

    Map<Object, Object> properties = getPlatform().getProperties();

    if( getPlatform().isMapReduce() )
      FlowConnectorProps.setIntermediateSchemeClass( properties, schemeClass );
View Full Code Here
165
166
167
168
169
170
171
172
173
174
    private NestedSubAssembly( Pipe pipe )
      {
      Pipe pipe1 = new Pipe( "second", pipe );

      Pipe pipe2 = new Pipe( "third", pipe );
      pipe2 = new Each( pipe2, new Identity() );
      pipe2 = new Pipe( "fourth", pipe2 );

      setTails( pipe1, pipe2 );
      }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    {
    Tap source = getPlatform().getTextFile( inputFileIps );

    Pipe pipe = new Pipe( "first" );

    pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/first" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  private Flow fourthFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "fourth" );

    pipe = new Each( pipe, new Identity() );

    Tap sink = getPlatform().getTextFile( getOutputPath( path + "/fourth" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
265
266
267
268
269
270
271
272
273
274
275
    Pipe splice = new GroupBy( "merge", Pipe.pipes( pipeLower, pipeUpper, pipeOffset ), new Fields( "num" ), new Fields( "char" ) );

    splice = new Every( splice, new Fields( "char" ), new First( new Fields( "first" ) ) );

    splice = new Each( splice, new Fields( "num", "first" ), new Identity() );

    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

    flow.complete();
View Full Code Here
461
462
463
464
465
466
467
468
469
470
471
    pipe = new Each( pipe, new Fields( "line" ), TestConstants.APACHE_COMMON_PARSER );

    pipe = new Each( pipe, new Fields( "method" ), new RegexFilter( "^POST" ) );
    pipe = new Each( pipe, new Fields( "method" ), new RegexFilter( "^POST" ) );

    pipe = new Each( pipe, new Fields( "method" ), new Identity( new Fields( "value" ) ), Fields.ALL );

    pipe = new GroupBy( pipe, new Fields( "value" ) );

    pipe = new Every( pipe, new Count(), new Fields( "value", "count" ) );
View Full Code Here
499
500
501
502
503
504
505
506
507
508
509
    pipe = new Each( pipe, new Fields( "method" ), new RegexFilter( "^fobar" ) ); // intentionally filtering all

    pipe = new GroupBy( pipe, new Fields( "method" ) );

    pipe = new Each( pipe, new Fields( "method" ), new Identity( new Fields( "value" ) ), Fields.ALL );

    pipe = new GroupBy( pipe, new Fields( "value" ) );

    pipe = new Every( pipe, new Count(), new Fields( "value", "count" ) );
View Full Code Here
782
783
784
785
786
787
788
789
790
791
792
793
    Pipe pipe = new Pipe( "test" );

    Function parser = new RegexParser( new Fields( 0 ), "^[^ ]*" );
    pipe = new Each( pipe, new Fields( "line" ), parser, Fields.REPLACE );
    pipe = new Each( pipe, new Fields( "line" ), new Identity( Fields.ARGS ), Fields.REPLACE );
    pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "line" ) ), Fields.REPLACE );

    pipe = new Each( pipe, new Debug( true ) );

    Flow flow = getPlatform().getFlowConnector( disableDebug() ).connect( source, sink, pipe );

View Full Code Here
TOP

Related Classes of cascading.operation.Identity

  • bixo.examples.crawl.DemoCrawlWorkflow
  • cascading.assembly.CrossTab
  • cascading.BasicPipesPlatformTest
  • cascading.cascade.CascadePlatformTest
  • cascading.cascade.hadoop.RiffleCascadePlatformTest
  • cascading.cascade.ParallelCascadePlatformTest
  • cascading.CoGroupFieldedPipesPlatformTest
  • cascading.DistanceUseCasePlatformTest
  • cascading.FieldedPipesPlatformTest
  • cascading.flow.hadoop.BuildJobsHadoopPlatformTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.