Examples of org.apache.hadoop.io.IntWritable$Comparator

org.apache.hadoop.io.IntWritable
A Comparator optimized for IntWritable.

    
    List<Integer> keys = new ArrayList<Integer>();
    List<Step0Output> values = new ArrayList<Step0Output>();
    
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
    
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
      
      try {
        while (reader.next(key, value)) {
          keys.add(key.get());
          values.add(value.clone());
        }
      } finally {
        reader.close();
      }

View Full Code Here

      size++;
    }
    
    @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
      context.write(new IntWritable(partition), new Step0Output(firstId, size));
    }

View Full Code Here

        firstId = new LongWritable(key.get());
      }
      
      Instance instance = converter.convert((int) key.get(), value.toString());
      
      context.write(firstId, new IntWritable(instance.label));
    }

View Full Code Here

      return value;
    }
    
    @Override
    public void initialize(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException {
      key = new IntWritable();
      value = NullWritable.get();
    }

View Full Code Here

    
    int[] keys = new int[numMaps];
    Step0Output[] values = new Step0Output[numMaps];
    
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
    
    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, job);
      
      try {
        while (reader.next(key, value)) {
          keys[index] = key.get();
          values[index] = value.clone();
          
          index++;
        }
      } finally {

View Full Code Here

    Path path = new Path(outPath, "largestCleanEigens");
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    VectorWritable vw = new VectorWritable();
    IntWritable iw = new IntWritable();
    for(Map.Entry<MatrixSlice,EigenStatus> pruneSlice : prunedEigenMeta) {
      MatrixSlice s = pruneSlice.getKey();
      EigenStatus meta = pruneSlice.getValue();
      EigenVector ev = new EigenVector((DenseVector)s.vector(),
                                       meta.getEigenValue(),
                                       Math.abs(1-meta.getCosAngle()),
                                       s.index());
      log.info("appending " + ev.getName() + " to " + path.toString());
      vw.set(ev);
      iw.set(s.index());
      seqWriter.append(iw, vw);
    }
    seqWriter.close();
  }

View Full Code Here

      size++;
    }
    
    @Override
    public void close() throws IOException {
      collector.collect(new IntWritable(partition), new Step0Output(firstId, size));
    }

View Full Code Here

    public void close() throws IOException {
    }
    
    @Override
    public IntWritable createKey() {
      return new IntWritable();
    }

View Full Code Here

    FileSystem fs = outputPath.getFileSystem(conf);
    
    Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath);
    
    // import the InMemOutputs
    IntWritable key = new IntWritable();
    MapredOutput value = new MapredOutput();
    
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
      
      try {
        while (reader.next(key, value)) {
          output.put(key.get(), value.clone());
        }
      } finally {
        reader.close();
      }
    }

View Full Code Here

                                                         path,
                                                         intWritable ? IntWritable.class : LongWritable.class,
                                                         VectorWritable.class);
    int recNum = 0;
    for (VectorWritable point : points) {
      writer.append(intWritable ? new IntWritable(recNum++) : new LongWritable(recNum++), point);
    }
    writer.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.IntWritable$Comparator

bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest

bbejeck.mapred.aggregation.TemperatureAveragingPair

bbejeck.mapred.coocurrance.PairsRelativeOccurrenceReducerTest

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.merge.TestMergeJob

co.nubetech.hiho.similarity.ngram.TestScoreMapper

co.nubetech.hiho.similarity.ngram.TestScoreReducer

com.alimama.mdrill.index.IndexReducer

com.datasalt.pangool.utils.HadoopUtils

com.digitalpebble.behemoth.BehemothDocument

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.