Package com.datasalt.pangool.tuplemr

Examples of com.datasalt.pangool.tuplemr.TupleMRBuilder.createJob()

87
88
89
90
91
92
93
94
95
96
97
        ITuple.class, NullWritable.class);
    job.setTupleReducer(new IdentityTupleReducer());
    // Add multi-output: English index
    job.setOutput(new Path(output), new TupleSolrOutputFormat(new File("src/test/resources/solr-en"),
        conf), ITuple.class, NullWritable.class);
    Job hadoopJob = job.createJob();
    try {
      hadoopJob.waitForCompletion(true);
      if(!hadoopJob.isSuccessful()) {
        throw new PangoolRuntimeException("Job was not sucessfull");
      }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
    builder.setTupleReducer(new IdentityRed());
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class), new Map());

    Job job = builder.createJob();
    try {
      job.setNumReduceTasks(1);
      assertRun(job);
    } finally {
      builder.cleanUpInstanceFiles();
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    builder.setTupleReducer(new IdentityRed());
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class), new Map());

    Job job = builder.createJob();
    try {
      job.setNumReduceTasks(1);
      assertRun(job);
    } finally {
      builder.cleanUpInstanceFiles();
View Full Code Here
455
456
457
458
459
460
461
462
463
464
465
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class),
        new DoNothingMap());

    Job job = builder.createJob();
    try {
      job.setNumReduceTasks(1);
      assertRun(job);
    } finally {
      builder.cleanUpInstanceFiles();
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
        assertEquals(false, ((ITuple)currentTuple.get("tuple2")).get("d"));
      };
    });
    builder.setGroupByFields("partitionId");
    builder.setOutput(new Path(OUTPUT), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    Job job = builder.createJob();
    try {
      job.waitForCompletion(true);
    } finally {
      builder.cleanUpInstanceFiles();
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    builder.setTupleReducer(new IdentityTupleReducer());
    builder.addIntermediateSchema(schema);
    builder.setGroupByFields(schema.getField(0).getName());
    builder.setTupleOutput(new Path(output), schema);

    Job job = builder.createJob();
    try {
      job.setNumReduceTasks(1);
      assertRun(job);
    } finally {
      builder.cleanUpInstanceFiles();
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    };
    job.setTupleCombiner(countReducer);
    job.setTupleReducer(countReducer);
    job.setGroupByFields("word", "category");
    job.setTupleOutput(new Path(output), INTERMEDIATE_SCHEMA);
    if(job.createJob().waitForCompletion(true)) {
      return 1;
    }
    return -1;
  }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
    mr.setOutput(outputPath,new AvroOutputFormat<Record>(getAvroOutputSchema()),
        AvroWrapper.class,NullWritable.class);

    mr.setTupleReducer(new Red());

    Job job = mr.createJob();
    job.waitForCompletion(true);

    return 0;
  }
 
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
    // We'll use a TupleOutputFormat with the same schema than the intermediate schema
    mr.setTupleOutput(new Path(args[1]), getSchema());
    mr.setTupleReducer(new CountReducer());
    mr.setTupleCombiner(new CountReducer());

    mr.createJob().waitForCompletion(true);

    return 1;
  }
 
  public static void main(String[] args) throws Exception {
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
      builder.setGroupByFields(groupBy);
      builder.setOutput(outP, outputFormat, ITuple.class, NullWritable.class);
      builder.setTupleReducer(new IdentityTupleReducer());
      builder.setJarByClass(this.getClass());
     
      builder.createJob().waitForCompletion(true);
    }

    return 1;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.