Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.ITuple

95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    try {
      Iterator<NullWritable> iterator = values.iterator();
      tupleIterator.setIterator(iterator);

      // We get the firts tuple, to create the groupTuple view
      ITuple firstTupleGroup = key.datum();

      // A view is created over the first tuple to give the user the group
      // fields
      if(isMultipleSources) {
        int schemaId = tupleMRConfig.getSchemaIdByName(firstTupleGroup.getSchema()
            .getName());
        int[] indexTranslation = serInfo.getGroupSchemaIndexTranslation(schemaId);
        groupTuple.setContained(firstTupleGroup, indexTranslation);
      } else {
        groupTuple.setContained(firstTupleGroup);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  public void open(OutputStream out) {
    tupleSerializer.open(out);
  }

  public void serialize(DatumWrapper<ITuple> wrapper) throws IOException {
    ITuple tuple = wrapper.datum();
    if (isMultipleSources) {
      multipleSourcesSerialization(tuple);
    } else {
      oneSourceSerialization(tuple);
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
    if(isRollup) {
      t.swapInstances();
      this.cachedTuples.swapInstances();
    }

    ITuple tuple = (multipleSources) ? deserializeMultipleSources() : deserializeOneSource(t.datum());
    t.datum(tuple);

    return t;
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    return t;
  }

  private ITuple deserializeMultipleSources() throws IOException {
    CachedTuples tuples = cachedTuples.datum();
    ITuple commonTuple = tuples.commonTuple;

    simpleTupleDeSer.readFields(commonTuple, serInfo.getCommonSchemaDeserializers());
    int schemaId = WritableUtils.readVInt(simpleTupleDeSer.getInput());
    ITuple specificTuple = tuples.specificTuples.get(schemaId);
    simpleTupleDeSer.readFields(specificTuple, serInfo.getSpecificSchemaDeserializers().get(schemaId));
    ITuple result = tuples.resultTuples.get(schemaId);
    mixIntermediateIntoResult(commonTuple, specificTuple, result, schemaId);
    return result;
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    }
  }

  private ITuple deserializeOneSource(ITuple reuse) throws IOException {
    CachedTuples tuples = cachedTuples.datum();
    ITuple commonTuple = tuples.commonTuple;
    simpleTupleDeSer.readFields(commonTuple, serInfo.getCommonSchemaDeserializers());
    if(reuse == null) {
      reuse = tuples.resultTuples.get(0);
    }
    int[] commonTranslation = serInfo.getCommonSchemaIndexTranslation(0); // just one common schema
    for(int i = 0; i < commonTranslation.length; i++) {
      int destPos = commonTranslation[i];
      reuse.set(destPos, commonTuple.get(i));
    }
    return reuse;
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    for(FileStatus fileStatus : fileSystem.globStatus(generatedModel)) {
      TupleInputReader reader = new TupleInputReader(conf);
      reader.initialize(fileStatus.getPath(), conf);
      while(reader.nextKeyValueNoSync()) {
        // Read Tuple
        ITuple tuple = reader.getCurrentKey();
        Integer count = (Integer) tuple.get("count");
        Category category = (Category) tuple.get("category");
        String word = tuple.get("word").toString();
        vocabulary.add(word);
        tokensPerCategory.put(category, MapUtils.getInteger(tokensPerCategory, category, 0) + count);
        wordCountPerCategory.get(category).put(word, count);
      }
    }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    TupleReducer countReducer = new TupleReducer<ITuple, NullWritable>() {

      public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
          throws IOException, InterruptedException, TupleMRException {
        int count = 0;
        ITuple outputTuple = null;
        for(ITuple tuple : tuples) {
          count += (Integer) tuple.get("count");
          outputTuple = tuple;
        }
        outputTuple.set("count", count);
        collector.write(outputTuple, NullWritable.get());
      }
    };
    job.setTupleCombiner(countReducer);
    job.setTupleReducer(countReducer);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
        throws java.io.IOException, InterruptedException, TupleMRException {

      int totalCount = 0;
      Iterator<ITuple> iterator = tuples.iterator();
      for(int i = 0; i < n && iterator.hasNext(); i++) {
        ITuple tuple = iterator.next();
        collector.write(tuple, NullWritable.get());
        totalCount += (Integer) tuple.get("count");
      }
     
      outputCountTuple.set("topic", group.get("topic"));
      outputCountTuple.set("totalcount", totalCount);
      collector.getNamedOutput(OUTPUT_TOTALCOUNT).write(outputCountTuple, NullWritable.get());
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
    @Override
    public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      int count = 0;
      ITuple outputTuple = null;
      for(ITuple tuple : tuples) {
        count += (Integer) tuple.get("count");
        outputTuple = tuple;
      }
      outputTuple.set("count", count);
      collector.write(outputTuple, NullWritable.get());
    }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    @Override
    public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      int count = 0;
      ITuple outputTuple = null;
      Record outputRecord=null;
      for(ITuple tuple : tuples) {
        Record record = (Record)tuple.get("my_avro");
        count += (Integer) record.get("count");
        outputTuple = tuple;
        outputRecord = record;
      }
      outputRecord.put("count",count);
      outputTuple.set("my_avro",outputRecord);
      collector.write(outputTuple, NullWritable.get());
    }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.ITuple

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$CountReducer
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesGenerate
  • com.datasalt.pangool.examples.topicalwordcount.TestTopicFingerprint
  • com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint$TopNWords
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleInputOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.RollupReducer
  • com.datasalt.pangool.tuplemr.mapred.SimpleCombiner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.