Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.ITuple

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    TupleReducer countReducer = new TupleReducer<ITuple, NullWritable>() {

      public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
          throws IOException, InterruptedException, TupleMRException {
        int count = 0;
        ITuple outputTuple = null;
        for(ITuple tuple : tuples) {
          count += (Integer) tuple.get("count");
          outputTuple = tuple;
        }
        outputTuple.set("count", count);
        collector.write(outputTuple, NullWritable.get());
      }
    };
    job.setTupleCombiner(countReducer);
    job.setTupleReducer(countReducer);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    @Override
    public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      int count = 0;
      ITuple outputTuple = null;
      Record outputRecord=null;
      for(ITuple tuple : tuples) {
        Record record = (Record)tuple.get("my_avro");
        count += (Integer) record.get("count");
        outputTuple = tuple;
        outputRecord = record;
      }
      outputRecord.put("count",count);
      outputTuple.set("my_avro",outputRecord);
      collector.write(outputTuple, NullWritable.get());
    }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
  }

  public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
      throws IOException, InterruptedException, TupleMRException {
    int totalCount = 0;
    ITuple outTuple = null;
    for(ITuple tuple : tuples) {
      totalCount += (Integer) tuple.get(field);
      outTuple = tuple;
    }
    outTuple.set(field, totalCount);
    collector.write(outTuple, NullWritable.get());
  }
View Full Code Here
261
262
263
264
265
266
267
268
269
270
271
272
273
    }
    return null;
  }

  public static ITuple getTupleWithNulls(String id, String value, Integer intValue, Double doubleValue, String strValue) {
    ITuple tuple = new Tuple(theSchema2);
    tuple.set("id", id);
    tuple.set("value", value);
    tuple.set("intValue", intValue);
    tuple.set("doubleValue", doubleValue);
    tuple.set("strValue", strValue);
    return tuple;
  }
View Full Code Here
271
272
273
274
275
276
277
278
279
280
    tuple.set("strValue", strValue);
    return tuple;
  }
 
  public static ITuple getTuple(String id, String value) {
    ITuple tuple = new Tuple(theSchema1);
    tuple.set("id", id);
    tuple.set("value", value);
    return tuple;
  }
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
                // Collect Tuples with decreasing probability
                // (http://en.wikipedia.org/wiki/Reservoir_sampling)
                protected void map(ITuple key, NullWritable value, Context context) throws IOException,
                    InterruptedException {
                  ITuple uTuple;
                  try {
                    uTuple = processor.process(key, counterInterface);
                  } catch(Throwable e) {
                    throw new RuntimeException(e);
                  }
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
461
462
        RecordProcessor processor = recordProcessorPerSplit.get(split);
        Text key = new Text();
        while(reader.nextKeyValue()) {
          //
          ITuple tuple = reader.getCurrentKey();

          ITuple uTuple;
          try {
            uTuple = processor.process(tuple, counterInterface);
          } catch(Throwable e) {
            throw new RuntimeException(e);
          }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Configuration conf = new Configuration();

    TupleFile.Writer writer = new TupleFile.Writer(FileSystem.get(conf), conf, new Path(INPUT + "_r"), schema);
    for(int i = 0; i < 10000; i++) {
      ITuple tuple = new Tuple(schema);
      tuple.set("id", i+"");
      // We save a number in the "foo" field which is consecutive: [0, 1, 2, ... 9999]
      tuple.set("foo", "foo" + i);
      writer.append(tuple);
    }
    writer.close();

    // Sampling with default method should yield lower numbers
View Full Code Here
168
169
170
171
172
173
174
175
176
177
  }

  final Schema schema = new Schema("schema", Fields.parse("id:string, foo:string"));

  public ITuple randomTuple() {
    ITuple tuple = new Tuple(schema);
    tuple.set("id", "id" + (Math.random() * 1000000000));
    tuple.set("foo", "foobar" + (Math.random() * 1000000000));
    return tuple;
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    try {
      Iterator<NullWritable> iterator = values.iterator();
      tupleIterator.setIterator(iterator);

      // We get the firts tuple, to create the groupTuple view
      ITuple firstTupleGroup = key.datum();

      // A view is created over the first tuple to give the user the group
      // fields
      if(isMultipleSources) {
        int schemaId = tupleMRConfig.getSchemaIdByName(firstTupleGroup.getSchema()
            .getName());
        int[] indexTranslation = serInfo.getGroupSchemaIndexTranslation(schemaId);
        groupTuple.setContained(firstTupleGroup, indexTranslation);
      } else {
        groupTuple.setContained(firstTupleGroup);
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.ITuple

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$CountReducer
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesGenerate
  • com.datasalt.pangool.examples.topicalwordcount.TestTopicFingerprint
  • com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint$TopNWords
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleInputOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.RollupReducer
  • com.datasalt.pangool.tuplemr.mapred.SimpleCombiner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.