Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.LongWritable$Comparator


    URI uri = FILE.toURI();
    SequenceFile.Writer writer
      = new SequenceFile.Writer(FileSystem.get(uri, c), c,
                                new Path(uri.toString()),
                                LongWritable.class, Text.class);
    final LongWritable key = new LongWritable();
    final Text val = new Text();
    for (int i = 0; i < COUNT; ++i) {
      key.set(i);
      val.set(Integer.toString(i));
      writer.append(key, val);
    }
    writer.close();
  }
View Full Code Here


                cbsyncs = 0L;
              }
            }

            Path dst = makeRelative(root, stat.getPath());
            src_writer.append(new LongWritable(stat.isDir()? 0: stat.getLen()),
                new FilePair(stat, dst));
            dst_writer.append(new Text(dst.toString()),
                new Text(stat.getPath().toString()));
          }
        }
View Full Code Here

      Path src = new Path(srcfilelist);
      FileSystem fs = src.getFileSystem(job);
      FileStatus srcst = fs.getFileStatus(src);

      ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
      LongWritable key = new LongWritable();
      FilePair value = new FilePair();
      final long targetsize = cbsize / numSplits;
      long pos = 0L;
      long last = 0L;
      long acc = 0L;
      long cbrem = srcst.getLen();
      for (SequenceFile.Reader sl = new SequenceFile.Reader(fs, src, job);
           sl.next(key, value); last = sl.getPosition()) {
        // if adding this split would put this split past the target size,
        // cut the last split and put this next file in the next split.
        if (acc + key.get() > targetsize && acc != 0) {
          long splitsize = last - pos;
          splits.add(new FileSplit(src, pos, splitsize, job));
          cbrem -= splitsize;
          pos = last;
          acc = 0L;
        }
        acc += key.get();
      }
      if (cbrem != 0) {
        splits.add(new FileSplit(src, pos, cbrem, job));
      }
View Full Code Here

    values.add(new Time(new Date().getTime()));
    values.add(new Time(new Date().getTime()));
    values.add(new Float(1.0f));

    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    StringBuilder builder = new StringBuilder();
View Full Code Here

    values.add(new Time(new Date().getTime()));
    values.add(new Time(new Date().getTime()));
    values.add(new Float(1.0f));

    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    StringBuilder builder = new StringBuilder();
View Full Code Here

    DBInputDelimMapper mapper = new DBInputDelimMapper();
    ArrayList<ColumnInfo> columns = new ArrayList<ColumnInfo>();
    ArrayList values = new ArrayList();
    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    verify(context).write(outkey, outval);
View Full Code Here

    assertEquals(md5HashKey1, md5HashKey2);
  }

  @Test
  public void testMD5HashForLongWritableKey() throws IOException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
  }
View Full Code Here

    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1,Sam,true,84.0"), context);
   
    ArrayList values = new ArrayList();
    values.add(1l);
    values.add("Sam");
    values.add(true);
View Full Code Here

    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1, ,true,84.0"), context);
   
    ArrayList values = new ArrayList();
    values.add(1l);
    values.add(null);
    values.add(true);
View Full Code Here

    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1,Sam,true,84.0,42"), context);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.LongWritable$Comparator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.