Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.NullWritable$Comparator

455
456
457
458
459
460
461
462
463
464
465
    assertEquals(1, splits.length);

    // read the whole file
    org.apache.hadoop.mapred.RecordReader<NullWritable, OrcLazyRow> reader =
        in.getRecordReader(splits[0], conf, Reporter.NULL);
    NullWritable key = reader.createKey();
    OrcLazyRow value = (OrcLazyRow) reader.createValue();
    List<? extends StructField> fields =inspector.getAllStructFieldRefs();
    StringObjectInspector strInspector = (StringObjectInspector)
        fields.get(0).getFieldObjectInspector();
    assertEquals(true, reader.next(key, value));
View Full Code Here
529
530
531
532
533
534
535
536
537
538
539
     * @throws IOException 読み込みに失敗した場合
     * @deprecated Use {@link #load(ModelInput)} instead
     */
    @Deprecated
    public void load(SequenceFile.Reader reader) throws IOException {
        NullWritable key = NullWritable.get();
        for (;;) {
            Writable model;
            try {
                model = modelClass.newInstance();
            } catch (InstantiationException e) {
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
  public static Vector retrieveTimesSquaredOutputVector(JobConf conf) throws IOException {
    Path outputPath = FileOutputFormat.getOutputPath(conf);
    FileSystem fs = FileSystem.get(conf);
    Path outputFile = new Path(outputPath, "part-00000");
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outputFile, conf);
    NullWritable n = NullWritable.get();
    VectorWritable v = new VectorWritable();
    reader.next(n,v);
    Vector vector = v.get();
    reader.close();
    fs.deleteOnExit(outputFile);
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
        SequenceFile.Reader reader = new SequenceFile.Reader(fs,
          inputVectorPath,
          conf);
        VectorWritable val = new VectorWritable();
        NullWritable nw = NullWritable.get();
        reader.next(nw, val);
        reader.close();
        inputVector = val.get();
        if(!(inputVector instanceof SequentialAccessSparseVector || inputVector instanceof DenseVector)) {
          inputVector = new SequentialAccessSparseVector(inputVector);
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
        InMemRecordReader reader = (InMemRecordReader) inputFormat.getRecordReader(
            split, conf, null);

        for (int tree = 0; tree < split.getNbTrees(); tree++) {
          IntWritable key = reader.createKey();
          NullWritable value = reader.createValue();

          // reader.next() should return true until there is no tree left
          assertEquals(tree < split.getNbTrees(), reader.next(key, value));
         
          assertEquals(split.getFirstId() + tree, key.get());
View Full Code Here
361
362
363
364
365
366
367
368
369
370
371
    if (fs.exists(dst)) {
      fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs,
      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for(int i = 1; i < numPartitions; ++i) {
      int k = Math.round(stepSize * i);
      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
        "totalordermemcmp", conf, splitStrings);
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : testStrings) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
        "totalorderbinarysearch", conf, splitStrings);
    conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : testStrings) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
    revCheck.add(new Check<Text>(new Text("z"), 0));
    revCheck.add(new Check<Text>(new Text("ddngo"), 4));
    revCheck.add(new Check<Text>(new Text("hi"), 3));
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : revCheck) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
  private Object[] readPartitions(FileSystem fs, Path p, Class<?> keyClass,
      Configuration conf) throws IOException {
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
    ArrayList<Object> parts = new ArrayList<Object>();
    Writable key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
    NullWritable value = NullWritable.get();
    while (reader.next(key, value)) {
      parts.add(key);
      key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
    }
    reader.close();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.NullWritable$Comparator

  • bulkimport.BulkImportJobExample$VerboseInputSampler
  • ca.ulaval.ift.graal.pagerank.mapreduce.TopKMapper
  • cc.bf.ReadNgram
  • codechicken.lib.vec.BlockCoord
  • com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest
  • com.asakusafw.testtools.TestDataHolder
  • com.ebay.erl.mobius.core.datajoin.EvenlyPartitioner
  • com.facebook.hive.orc.TestInputOutputFormat
  • com.thinkaurelius.faunus.FaunusVertexTest
  • com.twitter.maple.tap.MemorySourceTap$MemorySourceScheme
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.