Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.IntWritable$Comparator


   
    List<Integer> keys = new ArrayList<Integer>();
    List<Step0Output> values = new ArrayList<Step0Output>();
   
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
   
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
     
      try {
        while (reader.next(key, value)) {
          keys.add(key.get());
          values.add(value.clone());
        }
      } finally {
        reader.close();
      }
View Full Code Here


      size++;
    }
   
    @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
      context.write(new IntWritable(partition), new Step0Output(firstId, size));
    }
View Full Code Here

        firstId = new LongWritable(key.get());
      }
     
      Instance instance = converter.convert((int) key.get(), value.toString());
     
      context.write(firstId, new IntWritable(instance.label));
    }
View Full Code Here

      return value;
    }
   
    @Override
    public void initialize(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException {
      key = new IntWritable();
      value = NullWritable.get();
    }
View Full Code Here

   
    int[] keys = new int[numMaps];
    Step0Output[] values = new Step0Output[numMaps];
   
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
   
    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, job);
     
      try {
        while (reader.next(key, value)) {
          keys[index] = key.get();
          values[index] = value.clone();
         
          index++;
        }
      } finally {
View Full Code Here

    Path path = new Path(outPath, "largestCleanEigens");
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    VectorWritable vw = new VectorWritable();
    IntWritable iw = new IntWritable();
    for(Map.Entry<MatrixSlice,EigenStatus> pruneSlice : prunedEigenMeta) {
      MatrixSlice s = pruneSlice.getKey();
      EigenStatus meta = pruneSlice.getValue();
      EigenVector ev = new EigenVector((DenseVector)s.vector(),
                                       meta.getEigenValue(),
                                       Math.abs(1-meta.getCosAngle()),
                                       s.index());
      log.info("appending " + ev.getName() + " to " + path.toString());
      vw.set(ev);
      iw.set(s.index());
      seqWriter.append(iw, vw);
    }
    seqWriter.close();
  }
View Full Code Here

      size++;
    }
   
    @Override
    public void close() throws IOException {
      collector.collect(new IntWritable(partition), new Step0Output(firstId, size));
    }
View Full Code Here

    public void close() throws IOException {
    }
   
    @Override
    public IntWritable createKey() {
      return new IntWritable();
    }
View Full Code Here

    FileSystem fs = outputPath.getFileSystem(conf);
   
    Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath);
   
    // import the InMemOutputs
    IntWritable key = new IntWritable();
    MapredOutput value = new MapredOutput();
   
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
     
      try {
        while (reader.next(key, value)) {
          output.put(key.get(), value.clone());
        }
      } finally {
        reader.close();
      }
    }
View Full Code Here

                                                         path,
                                                         intWritable ? IntWritable.class : LongWritable.class,
                                                         VectorWritable.class);
    int recNum = 0;
    for (VectorWritable point : points) {
      writer.append(intWritable ? new IntWritable(recNum++) : new LongWritable(recNum++), point);
    }
    writer.close();
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.IntWritable$Comparator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.