Package com.datasalt.pangool.tuplemr

Examples of com.datasalt.pangool.tuplemr.TupleMRBuilder.createJob()

143
144
145
146
147
148
149
150
151
152
153
    mr.setOrderBy(new OrderBy().add("url", Order.ASC).addSchemaOrder(Order.ASC));
    mr.setTupleReducer(new Handler());
    mr.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    mr.addInput(new Path(input1), new HadoopInputFormat(TextInputFormat.class), new UrlMapProcessor());
    mr.addInput(new Path(input2), new HadoopInputFormat(TextInputFormat.class), new UrlProcessor());
    mr.createJob().waitForCompletion(true);

    return 1;
  }
 
  public static void main(String[] args) throws Exception {
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
    // We'll use a TupleOutputFormat with the same schema than the intermediate schema
    mr.setTupleOutput(new Path(args[1]), getSchema());
    mr.setTupleReducer(new CountReducer());
    mr.setTupleCombiner(new CountReducer());

    mr.createJob().waitForCompletion(true);

    return 1;
  }
 
  public static void main(String[] args) throws Exception {
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
    mr.setOutput(outputPath,new AvroOutputFormat<Record>(getAvroOutputSchema()),
        AvroWrapper.class,NullWritable.class);

    mr.setTupleReducer(new Red());

    Job job = mr.createJob();
    job.waitForCompletion(true);

    return 0;
  }
 
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
        NullWritable.class);

    mr.setTupleReducer(new Red());

    try {
      Job job = mr.createJob();
      job.waitForCompletion(true);
    } finally {
      mr.cleanUpInstanceFiles();
    }
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
    mr.setTupleOutput(new Path(args[1]), getSchema());
    mr.setTupleReducer(new CountReducer());
    mr.setTupleCombiner(new CountReducer());

    try {
      mr.createJob().waitForCompletion(true);
    } finally {
      mr.cleanUpInstanceFiles();
    }
    return 1;
  }
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
        NullWritable.class);
    mr.addInput(new Path(input1), new HadoopInputFormat(TextInputFormat.class), new UrlMapProcessor());
    mr.addInput(new Path(input2), new HadoopInputFormat(TextInputFormat.class), new UrlProcessor());

    try {
      mr.createJob().waitForCompletion(true);
    } finally {
      mr.cleanUpInstanceFiles();
    }

    return 1;
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
        }
      }
    });

    try {
      Job hadoopJob = job.createJob();
      hadoopJob.waitForCompletion(true);
    } finally {
      job.cleanUpInstanceFiles();
    }
    return 0;
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
    mr.setOrderBy(new OrderBy().add("url", Order.ASC).addSchemaOrder(Order.ASC));
    mr.setTupleReducer(new Handler());
    mr.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    mr.addInput(new Path(input1), new HadoopInputFormat(TextInputFormat.class), new UrlMapProcessor());
    mr.addInput(new Path(input2), new HadoopInputFormat(TextInputFormat.class), new UrlProcessor());
    mr.createJob().waitForCompletion(true);

    return 1;
  }
 
  public static void main(String[] args) throws Exception {
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    builder.setOrderBy(new OrderBy().add("first",Order.ASC).add("second",Order.ASC));
    // Input / output and such
    builder.setTupleReducer(new Handler());
    builder.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    builder.addInput(new Path(input), new HadoopInputFormat(TextInputFormat.class), new IProcessor());
    builder.createJob().waitForCompletion(true);

    return 1;
  }
 
  public static void main(String args[]) throws Exception {
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
          collector.getNamedOutput(category.toString().toLowerCase()).write(outTuple, NullWritable.get());
        }
      }
    });

    Job hadoopJob = job.createJob();
    hadoopJob.waitForCompletion(true);
    return 0;
  }
 
  public MultiShakespeareIndexer() {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.