Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.Tuple

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
  private static CachedTuples createCachedTuples(TupleMRConfig config){
    SerializationInfo serInfo = config.getSerializationInfo();
    boolean multipleSources = config.getNumIntermediateSchemas() >= 2;
    CachedTuples r = new CachedTuples();
    r.commonTuple = new Tuple(serInfo.getCommonSchema());
    for (Schema sourceSchema : config.getIntermediateSchemas()){
      r.resultTuples.add(new Tuple(sourceSchema));
    }
   
    if (multipleSources){
      for(Schema specificSchema : serInfo.getSpecificSchemas()){
        r.specificTuples.add(new Tuple(specificSchema));
      }
    }
    return r;
  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
   * Moves data between a Record and a Tuple
   */
  public ITuple toTuple(Record record, ITuple reuse) throws IOException {
    ITuple tuple = reuse;
    if (tuple == null){
      tuple = new Tuple(pangoolSchema);
    }
   
    Schema pangoolSchema = tuple.getSchema();
    for(org.apache.avro.Schema.Field avroField : avroSchema.getFields()) {
      int pos = avroField.pos();
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
      int newSize = 0;
      if(line == null) {
        this.line = new Text();
      }
      if(tuple == null) {
        this.tuple = new Tuple(schema);
      }
      while(position < end) {
        newSize = in.readLine(line, maxLineLength,
            Math.max((int) Math.min(Integer.MAX_VALUE, end - position), maxLineLength));
View Full Code Here
66
67
68
69
70
71
72
73
74
75
  }

  @Override
  public ITuple deserialize(ITuple tuple) throws IOException {
    if(tuple == null) {
      tuple = new Tuple(schemaToDeserialize);
    }
    readFields(tuple, deserializers);
    return tuple;
  }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    // Use a HashSet to calculate the total vocabulary size
    Set<String> vocabulary = new HashSet<String>();
    // Read tuples from generate job
    for(FileStatus fileStatus : fileSystem.globStatus(generatedModel)) {
      TupleFile.Reader reader = new TupleFile.Reader(fileSystem, conf, fileStatus.getPath());
      Tuple tuple = new Tuple(reader.getSchema());
      while(reader.next(tuple)) {
        // Read Tuple
        Integer count = (Integer) tuple.get("count");
        Category category = (Category) tuple.get("category");
        String word = tuple.get("word").toString();
        vocabulary.add(word);
        tokensPerCategory.put(category, MapUtils.getInteger(tokensPerCategory, category, 0) + count);
        wordCountPerCategory.get(category).put(word, count);
      }
    }
View Full Code Here
64
65
66
67
68
69
70
  private static class TweetsMapper extends TupleMapper<AvroWrapper<Record>,NullWritable> {

    private Tuple tuple;
    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("tweet"));
    };
View Full Code Here
81
82
83
84
85
86
87
  @SuppressWarnings("serial")
  private static class RetweetsMapper extends TupleMapper<LongWritable,Text> {
    private Tuple tuple;
    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("retweet"));
    };
View Full Code Here
74
75
76
77
78
79
80
81
82
    protected ObjectMapper mapper;

    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
      record = new Record(getAvroSchema());
      tuple.set("my_avro",record);
    };
View Full Code Here
64
65
66
67
68
69
70
    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException,
        InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("tweet"));
    };
View Full Code Here
82
83
84
85
86
87
88
  private static class RetweetsMapper extends TupleMapper<LongWritable, Text> {
    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException,
        InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("retweet"));
    };
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Tuple

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$RetweetsMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.movingaverage.MovingAverage$URLVisitsProcessor
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesGenerate
  • com.datasalt.pangool.examples.secondarysort.SecondarySort$IProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.