Package com.datasalt.pangool.tuplemr.mapred.lib.input

Examples of com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat

76
77
78
79
80
81
82
83
84
85
86
    writer.close();

    TablespaceBuilder builder = new TablespaceBuilder();
    builder.setNPartitions(3);
    TableBuilder tableBuilder = new TableBuilder(theSchema1);
    tableBuilder.addFile(new TableInput(new TupleInputFormat(), new HashMap<String, String>(),
        theSchema1, new IdentityRecordProcessor(), new Path(INPUT)));
    // Partition by a javascript that returns the first two characters
    tableBuilder
        .partitionByJavaScript("function partition(record) { var str = record.get('id').toString(); return str.substring(0, 2); }");
    builder.add(tableBuilder.build());
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
      tuple.set("s", r.nextLong() + "");
      writer.append(tuple);
    }
    writer.close();

    TupleInputFormat format = ReflectionUtils.newInstance(TupleInputFormat.class, getConf());
    Job job = new Job(getConf());
    FileInputFormat.setInputPaths(job, outPath);
    logger.info("Using max input split size: " + maxSplitSize);
    FileInputFormat.setMaxInputSplitSize(job, maxSplitSize);
    job.setInputFormatClass(FileInputFormat.class);

    // Read all the splits and count. The number of read rows must
    // be the same than the written ones.
    int count = 0;
    for(InputSplit split : format.getSplits(job)) {
      TaskAttemptID attemptId = new TaskAttemptID(new TaskID(), 1);
      TaskAttemptContext attemptContext = TaskAttemptContextFactory.get(getConf(), attemptId);
      logger.info("Sampling split: " + split);
      RecordReader<ITuple, NullWritable> reader = format.createRecordReader(split, attemptContext);
      reader.initialize(split, attemptContext);
      while(reader.nextKeyValue()) {
        tuple = reader.getCurrentKey();
        count++;
      }
View Full Code Here
147
148
149
150
151
152
153
   * Defines an input as in {@link PangoolMultipleInputs} using {@link TupleInputFormat}
   *
   * @see PangoolMultipleInputs
   */
  public void addTupleInput(Path path, TupleMapper<ITuple, NullWritable> tupleMapper) {
    multipleInputs.getMultiInputs().add(new Input(path, new TupleInputFormat(), tupleMapper));
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    writer.append(getTuple("id8", "value61"));
    writer.append(getTuple("id8", "value62"));
   
    writer.close();
   
    TablespaceSpec tablespace = TablespaceSpec.of(theSchema1, "id", new Path(INPUT), new TupleInputFormat(),  4);
    TablespaceGenerator viewGenerator = new TablespaceGenerator(tablespace, new Path(OUTPUT), this.getClass());
    viewGenerator.generateView(conf, SamplingType.FULL_SCAN, new TupleSampler.RandomSamplingOptions());
   
    List<PartitionEntry> partitionMap = viewGenerator.getPartitionMap().getPartitionEntries();
    assertEquals(4, partitionMap.size());
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    writer.append(getTupleWithNulls("id4", "value14", 100, 2.0, ""));
    writer.append(getTupleWithNulls("id5", "value15", 100, 2.0, null));
   
    writer.close();
   
    TablespaceSpec tablespace = TablespaceSpec.of(theSchema2, "id", new Path(INPUT), new TupleInputFormat(), 1);
    TablespaceGenerator viewGenerator = new TablespaceGenerator(tablespace, new Path(OUTPUT), this.getClass());
    viewGenerator.generateView(conf, SamplingType.FULL_SCAN, new TupleSampler.RandomSamplingOptions());
   
    SQLite4JavaClient manager = new SQLite4JavaClient(OUTPUT + "/store/0.db", null);
    String results = manager.query("SELECT * FROM schema2;", 100);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
        Arrays.asList(
            new Table(
                Arrays.asList(
                    new TableInput[] {
                        new TableInput(
                            new TupleInputFormat(),
                            new HashMap<String, String>(),
                            schema,
                            new IdentityRecordProcessor(),
                            new Path(INPUT + "_" + inputPostfix)) }),
                new TableSpec(
View Full Code Here
178
179
180
181
182
183
184
185
186
   * {@link TupleInputFormat}
   *
   * @see PangoolMultipleInputs
   */
  public void addTupleInput(Path path, TupleMapper<ITuple, NullWritable> tupleMapper) {
    this.multiInputs.add(new Input(path, new TupleInputFormat(), tupleMapper));
    AvroUtils.addAvroSerialization(conf);

  }
View Full Code Here
182
183
184
185
186
187
188
189
190
   * {@link TupleInputFormat}
   *
   * @see PangoolMultipleInputs
   */
  public void addTupleInput(Path path, TupleMapper<ITuple, NullWritable> tupleMapper) {
    this.multiInputs.add(new Input(path, new TupleInputFormat(), tupleMapper));
    AvroUtils.addAvroSerialization(conf);

  }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat

  • com.datasalt.pangool.io.Tuple
  • com.datasalt.pangool.serialization.HadoopSerialization
  • com.datasalt.pangool.tuplemr.MapOnlyJobBuilder
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleInputOutputFormat
  • com.datasalt.pangool.tuplemr.TupleMRBuilder
  • com.datasalt.pangool.utils.AvroRecordToTupleConverter
  • com.splout.db.hadoop.TableBuilder
  • com.splout.db.hadoop.TestTablespaceGenerator
  • com.splout.db.hadoop.TestTablespaceGeneratorJavaScript
  • com.splout.db.hadoop.TestTupleSampler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.