Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.BytesWritable


    byte[] buffer = new byte[size];
    for (int i = 0; i < buffer.length; i++) {
      buffer[i] = (byte)'A';
    }
    BytesWritable bytes = new BytesWritable(buffer);
    List<WALEntry<BytesWritable>> batch = Lists.newArrayList();
    long seqid = 0;
    long numBytes = 0;
    long count = 0;
    long start = System.currentTimeMillis();
View Full Code Here


        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId,
                                    mapId, i,
                                    splits[i].getClass().getName(),
                                    split);
          JobConf localConf = new JobConf(job);
View Full Code Here

            Text t = (Text) from;
            to.bytes(t.getBytes(), t.getLength());
            return;
        }
        if (from instanceof BytesWritable) {
            BytesWritable b = (BytesWritable) from;
            to.bytes(b.getBytes(), b.getLength());
            return;
        }

        super.convert(from, to);
    }
View Full Code Here

    assertEquals(md5HashKey1, md5HashKey2);
  }

  @Test
  public void testMD5HashForBytesWritableKey() throws IOException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
  }
View Full Code Here

  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
View Full Code Here

  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
View Full Code Here

  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
View Full Code Here

  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
View Full Code Here

    for(String outputFile : getAllFiles(dir)) {
      String name = (new File(outputFile)).getName();
      if(name.startsWith(prefix)) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
        LongWritable key = new LongWritable();
        BytesWritable value = new BytesWritable();
        while(reader.next(key, value)) {
          String body = new String(value.getBytes(), 0, value.getLength());
          if (bodies.contains(body)) {
            LOG.debug("Found event body: {}", body);
            bodies.remove(body);
            found++;
          }
View Full Code Here

        System.arraycopy(content, eol + 1, raw, 0, raw.length);
       
        // populate key and values with the header and raw content.
        Text keyText = (Text)key;
        keyText.set(header);
        BytesWritable valueBytes = (BytesWritable)value;
        valueBytes.set(raw, 0, raw.length);

        // TODO: It would be best to start at the end of the gzip read but
        // the bytes read in gzip don't match raw bytes in the file so we
        // overshoot the next header.  With this current method you get
        // some false positives but don't miss records.
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.BytesWritable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.