Examples of org.apache.hadoop.io.NullWritable$Comparator

org.apache.hadoop.io.NullWritable
A Comparator "optimized" for NullWritable.

    final ArrayList<DoubleVector> centers = new ArrayList<DoubleVector>();
    SequenceFile.Reader reader = null;
    try {
      reader = new SequenceFile.Reader(fs, centroids, peer.getConfiguration());
      VectorWritable key = new VectorWritable();
      NullWritable value = NullWritable.get();
      while (reader.next(key, value)) {
        DoubleVector center = key.getVector();
        centers.add(center);
      }
    } catch (IOException e) {

View Full Code Here

    final int[] summationCount = new int[centers.length];


    // if our cache is not enabled, iterate over the disk items
    if (cache == null) {
      // we have an assignment step
      final NullWritable value = NullWritable.get();
      final VectorWritable key = new VectorWritable();
      while (peer.readNext(key, value)) {
        assignCentersInternal(newCenterArray, summationCount, key.getVector()
            .deepCopy());
      }
    } else {
      // if our cache is enabled but empty, we have to read it from disk first
      if (cache.isEmpty()) {
        final NullWritable value = NullWritable.get();
        final VectorWritable key = new VectorWritable();
        while (peer.readNext(key, value)) {
          DoubleVector deepCopy = key.getVector().deepCopy();
          cache.add(deepCopy);
          // but do the assignment directly

View Full Code Here

  }


  private void recalculateAssignmentsAndWrite(
      BSPPeer<VectorWritable, NullWritable, IntWritable, VectorWritable, CenterMessage> peer)
      throws IOException {
    final NullWritable value = NullWritable.get();
    // also use our cache to speed up the final writes if exists
    if (cache == null) {
      final VectorWritable key = new VectorWritable();
      IntWritable keyWrite = new IntWritable();
      while (peer.readNext(key, value)) {

View Full Code Here

      fs.delete(center, true);


    if (fs.exists(in))
      fs.delete(in, true);


    final NullWritable value = NullWritable.get();


    Writer centerWriter = new SequenceFile.Writer(fs, conf, center,
        VectorWritable.class, NullWritable.class);


    final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf,

View Full Code Here

      fs.delete(in, true);


    final SequenceFile.Writer centerWriter = SequenceFile.createWriter(fs,
        conf, center, VectorWritable.class, NullWritable.class,
        CompressionType.NONE);
    final NullWritable value = NullWritable.get();


    final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf,
        in, VectorWritable.class, NullWritable.class, CompressionType.NONE);


    Random r = new Random();

View Full Code Here

    FileStatus[] listStatus = fs.listStatus(outputPath);
    for (FileStatus status : listStatus) {
      if (!status.isDir()) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs,
            status.getPath(), conf);
        NullWritable key = NullWritable.get();
        DoubleWritable value = new DoubleWritable();
        if (reader.next(key, value)) {
          LOG.info("Output File: " + status.getPath());
          LOG.info("key: '" + key + "' value: '" + value + "' expected: '"
              + expectedResult + "'");

View Full Code Here

        NullWritable, NullWritable, NullWritable,
        NullWritable> immutableClassesGiraphConfiguration =
        new ImmutableClassesGiraphConfiguration<
            NullWritable, NullWritable, NullWritable, NullWritable>(
            nullConf);
    NullWritable vertexValue =
        immutableClassesGiraphConfiguration.createVertexValue();
    NullWritable edgeValue =
        immutableClassesGiraphConfiguration.createEdgeValue();
    NullWritable messageValue =
        immutableClassesGiraphConfiguration.createMessageValue();
    assertSame(vertexValue.getClass(), NullWritable.class);
    assertSame(vertexValue, edgeValue);
    assertSame(edgeValue, messageValue);
  }

View Full Code Here

  private K[] readPartitions(FileSystem fs, Path p, Class<K> keyClass,
      Configuration conf) throws IOException {
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
    ArrayList<K> parts = new ArrayList<K>();
    K key = ReflectionUtils.newInstance(keyClass, conf);
    NullWritable value = NullWritable.get();
    while (reader.next(key, value)) {
      parts.add(key);
      key = ReflectionUtils.newInstance(keyClass, conf);
    }
    reader.close();

View Full Code Here

    if (fs.exists(dst)) {
      fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, 
      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for(int i = 1; i < numPartitions; ++i) {
      int k = Math.round(stepSize * i);
      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {

View Full Code Here

    if (fs.exists(dst)) {
      fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, 
      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for(int i = 1; i < numPartitions; ++i) {
      int k = Math.round(stepSize * i);
      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.NullWritable$Comparator

bulkimport.BulkImportJobExample$VerboseInputSampler

ca.ulaval.ift.graal.pagerank.mapreduce.TopKMapper

cc.bf.ReadNgram

codechicken.lib.vec.BlockCoord

com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest

com.asakusafw.testtools.TestDataHolder

com.ebay.erl.mobius.core.datajoin.EvenlyPartitioner

com.facebook.hive.orc.TestInputOutputFormat

com.thinkaurelius.faunus.FaunusVertexTest

com.twitter.maple.tap.MemorySourceTap$MemorySourceScheme

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.