Package cascading.operation

Examples of cascading.operation.Identity

349
350
351
352
353
354
355
356
357
358
359
    Tap source = getPlatform().getTextFile( inputFileUnexpectedEndOfFile );
    Tap sink = getPlatform().getTextFile( getOutputPath( getTestName() ), SinkMode.REPLACE );

    Pipe pipe = new Pipe( "data" );
    pipe = new Each( pipe, new Identity() );

    FlowDef flowDef = FlowDef.flowDef().addSource( pipe, source ).addTailSink( pipe, sink );
    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

    try
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    {
    public FirstAssembly( Pipe previous )
      {
      Pipe pipe = new Pipe( "first", previous );

      pipe = new Each( pipe, new Identity() );

      pipe = new GroupBy( pipe, Fields.ALL );

      pipe = new Every( pipe, new First(), Fields.RESULTS );
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
    {
    public SecondAssembly( Pipe previous )
      {
      Pipe pipe = new Pipe( "second", previous );

      pipe = new Each( pipe, new Identity() );

      pipe = new FirstAssembly( pipe );

      setTails( pipe );
      }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( Fields.UNKNOWN ) );

    pipe = new Each( pipe, new Debug() );

    pipe = new Each( pipe, new Fields( 2 ), new Identity( new Fields( "label" ) ) );

    pipe = new Each( pipe, new Debug() );

    pipe = new Each( pipe, new Fields( "label" ), new RegexFilter( "[A-Z]*" ) );

View Full Code Here
81
82
83
84
85
86
87
88
89
    Fields sinkFields = new Fields( "third", "fourth" );
    Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE );

    Pipe pipe = new Pipe( "test" );
    pipe = new Each( pipe, new Fields( "third" ), new Identity() );

    verify( source, sink, pipe );
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
    Fields sinkFields = new Fields( "third", "fourth" );
    Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE );

    Pipe pipe = new Pipe( "test" );
    pipe = new Each( pipe, new Fields( "first" ), new Identity( new Fields( "none" ) ), new Fields( "third" ) );

    verify( source, sink, pipe );
    }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    // name and rate against others of same movie
    pipe = new CoGroup( pipe, new Fields( "movie" ), 1, new Fields( "name1", "movie", "rate1", "name2", "movie2", "rate2" ) );

    // remove useless fields
    pipe = new Each( pipe, new Fields( "movie", "name1", "rate1", "name2", "rate2" ), new Identity() );

    // remove lines if the names are the same
    pipe = new Each( pipe, new RegexFilter( "^[^\\t]*\\t([^\\t]*)\\t[^\\t]*\\t\\1\\t.*", true ) );

    // transpose values in fields by natural sort order
    pipe = new Each( pipe, new SortElements( new Fields( "name1", "rate1" ), new Fields( "name2", "rate2" ) ) );

    // unique the pipe
    pipe = new GroupBy( pipe, Fields.ALL );
    pipe = new Every( pipe, Fields.ALL, new First(), Fields.RESULTS );

    // calculate square of diff
    Function sqDiff = new Identity( new Fields( "score" ) )
    {
    public void operate( FlowProcess flowProcess, FunctionCall functionCall )
      {
      TupleEntry input = functionCall.getArguments();
      functionCall.getOutputCollector().add( new Tuple( Math.pow( input.getTuple().getDouble( 0 ) - input.getTuple().getDouble( 1 ), 2 ) ) );
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    // name and rate against others of same movie
    pipe = new CoGroup( pipe, new Fields( "movie" ), 1, new Fields( "name1", "movie", "rate1", "name2", "movie2", "rate2" ) );

    // remove useless fields
    pipe = new Each( pipe, new Fields( "movie", "name1", "rate1", "name2", "rate2" ), new Identity() );

    // remove lines if the names are the same
    pipe = new Each( pipe, new RegexFilter( "^[^\\t]*\\t([^\\t]*)\\t[^\\t]*\\t\\1\\t.*", true ) );

    // transpose values in fields by natural sort order
    pipe = new Each( pipe, new SortElements( new Fields( "name1", "rate1" ), new Fields( "name2", "rate2" ) ) );

    // unique the pipe
    pipe = new GroupBy( pipe, Fields.ALL );

    pipe = new Every( pipe, Fields.ALL, new First(), Fields.RESULTS );

    // calculate square of diff
    Function sqDiff = new Identity( new Fields( "score" ) )
    {
    public void operate( FlowProcess flowProcess, FunctionCall functionCall )
      {
      TupleEntry input = functionCall.getArguments();
      functionCall.getOutputCollector().add( new Tuple( Math.pow( input.getTuple().getDouble( 0 ) - input.getTuple().getDouble( 1 ), 2 ) ) );
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
    Tap source = new Hfs( new CommentScheme( new Fields( "line" ) ), inputFileComments );

    Pipe pipe = new Pipe( "test" );

    pipe = new Each( pipe, new Identity() );

    Tap sink = new Hfs( new TextLine( 1 ), getOutputPath( "testnulls" ), SinkMode.REPLACE );

    Flow flow = getPlatform().getFlowConnector( getProperties() ).connect( source, sink, pipe );
View Full Code Here
61
62
63
64
65
66
67
      throw new IllegalArgumentException( "fields arguments must be same size, from: " + fromFields.printVerbose() + " to: " + toFields.printVerbose() );

    if( !toFields.isDefined() )
      throw new IllegalArgumentException( "toFields must define field names: " + toFields.printVerbose() );

    setTails( new Each( previous, fromFields, new Identity( toFields ), Fields.SWAP ) );
    }
View Full Code Here
TOP

Related Classes of cascading.operation.Identity

  • bixo.examples.crawl.DemoCrawlWorkflow
  • cascading.assembly.CrossTab
  • cascading.BasicPipesPlatformTest
  • cascading.cascade.CascadePlatformTest
  • cascading.cascade.hadoop.RiffleCascadePlatformTest
  • cascading.cascade.ParallelCascadePlatformTest
  • cascading.CoGroupFieldedPipesPlatformTest
  • cascading.DistanceUseCasePlatformTest
  • cascading.FieldedPipesPlatformTest
  • cascading.flow.hadoop.BuildJobsHadoopPlatformTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.