Examples of IdentityTupleMapper

Examples of com.datasalt.pangool.tuplemr.IdentityTupleMapper

681
682
683
684
685
686
687
688
689
690
691
    InputFormat inputFormat = new TupleTextInputFormat(schema, fieldsPos, false, null);
    OutputFormat outputFormat = new TupleTextOutputFormat(schema, false, ' ',
        TupleTextOutputFormat.NO_QUOTE_CHARACTER, TupleTextOutputFormat.NO_ESCAPE_CHARACTER);

    builder.addInput(inPath, inputFormat, new IdentityTupleMapper());
    builder.setTupleReducer(new IdentityTupleReducer());
    builder.setOutput(outPath, outputFormat, ITuple.class, NullWritable.class);
    Job job = builder.createJob();
    try {
      assertRun(job);
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.IdentityTupleMapper

138
139
140
141
142
143
144
145
146
147
148
    // To make things nicer, we evolve the Schema and use a different Schema for reading the Tuple File.
    // We remove the "content" and add a new nullable field.
    Schema evolvedSchema = new Schema("evolved", Fields.parse("content:string, new_field:string?"));

    builder = new TupleMRBuilder(conf);
    builder.addTupleInput(outPath, evolvedSchema, new IdentityTupleMapper());
    builder.addIntermediateSchema(evolvedSchema);
    builder.setGroupByFields("content");
    builder.setTupleReducer(new MyGroupHandler());
    builder.setOutput(outPathText, new HadoopOutputFormat(TextOutputFormat.class), Text.class,
        NullWritable.class);
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.IdentityTupleMapper

56
57
58
59
60
61
62
63
64
65
66
    for(int i = 0; i < NUM_ROWS_TO_GENERATE; i++) {
      withTupleInput(input, fillTuple(true, tuple));
    }

    TupleMRBuilder builder = new TupleMRBuilder(getConf(), "test");
    builder.addTupleInput(new Path(input), new IdentityTupleMapper());
    builder.setTupleReducer(new IdentityTupleReducer());
    builder.addIntermediateSchema(schema);
    builder.setGroupByFields(schema.getField(0).getName());
    builder.setTupleOutput(new Path(output), schema);
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.IdentityTupleMapper

132
133
134
135
136
137
138
139
140
141
142
          separator.charAt(0), quotes.charAt(0), escape.charAt(0), FieldSelector.NONE, null);
      TupleTextOutputFormat outputFormat = new TupleTextOutputFormat(schema, false, separator.charAt(0),
          quotes.charAt(0), escape.charAt(0));

      builder.addIntermediateSchema(schema);
      builder.addInput(inputP, parsingInputFormat, new IdentityTupleMapper());
      builder.setGroupByFields(groupBy);
      builder.setOutput(outP, outputFormat, ITuple.class, NullWritable.class);
      builder.setTupleReducer(new IdentityTupleReducer());
      builder.setJarByClass(this.getClass());
     
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.