Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.Tuple

74
75
76
77
78
79
80
81
82
    protected ObjectMapper mapper;

    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
      record = new Record(getAvroSchema());
      tuple.set("my_avro",record);
    };
View Full Code Here
59
60
61
62
63
64
65
    }

    this.start = in.getPosition();
    more = start < end;

    tuple = new Tuple(in.getSchema());
  }
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
      int newSize = 0;
      if(line == null) {
        this.line = new Text();
      }
      if(tuple == null) {
        this.tuple = new Tuple(schema);
      }
      while(position < end) {
        newSize = in.readLine(line, maxLineLength,
            Math.max((int) Math.min(Integer.MAX_VALUE, end - position), maxLineLength));
View Full Code Here
75
76
77
78
79
80
81
82
83
84
  }

  @Override
  public ITuple deserialize(ITuple tuple) throws IOException {
    if(tuple == null) {
      tuple = new Tuple(schemaToDeserialize);
    }
    readFields(tuple, deserializers);
    return tuple;
  }
View Full Code Here
52
53
54
55
56
57
58
  private static class IProcessor extends TupleMapper<LongWritable, Text> {

    private Tuple tuple ;
   
    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("my_schema"));
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
    protected ObjectMapper mapper;

    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
      record = new Record(getAvroSchema());
      tuple.set("my_avro", record);
    }
View Full Code Here
55
56
57
58
59
60
61
    protected ObjectMapper mapper;

    public void setup(TupleMRContext context, Collector collector) throws IOException,
        InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
    };
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    @Override
    public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {
      if(result == null) {
        result = new Tuple(getURLRegisterSchema());
      }
      String cannonicalUrl = null;
      for(ITuple tuple : tuples) {
        if("urlMap".equals(tuple.getSchema().getName())) {
          cannonicalUrl = tuple.getString("canonicalUrl");
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
    @Override
    public void map(LongWritable key, Text value, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {

      // Just parsing the text input and emitting a Tuple
      Tuple tuple = new Tuple(schema);
      String[] fields = value.toString().trim().split("\t");
      tuple.set("url", fields[0]);
      tuple.set("date", fields[1]);
      tuple.set("visits", Integer.parseInt(fields[2]));
      collector.write(tuple);
    }
View Full Code Here
61
62
63
64
65
66
67
    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException {
      jsonMapper = new ObjectMapper();
      jsonMapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("my_schema"));
    }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Tuple

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$RetweetsMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.movingaverage.MovingAverage$URLVisitsProcessor
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesGenerate
  • com.datasalt.pangool.examples.secondarysort.SecondarySort$IProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.