Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.BytesWritable

252
253
254
255
256
257
258
259
260
261
262
    byte[] buffer = new byte[size];
    for (int i = 0; i < buffer.length; i++) {
      buffer[i] = (byte)'A';
    }
    BytesWritable bytes = new BytesWritable(buffer);
    List<WALEntry<BytesWritable>> batch = Lists.newArrayList();
    long seqid = 0;
    long numBytes = 0;
    long count = 0;
    long start = System.currentTimeMillis();
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId,
                                    mapId, i,
                                    splits[i].getClass().getName(),
                                    split);
          JobConf localConf = new JobConf(job);
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
            Text t = (Text) from;
            to.bytes(t.getBytes(), t.getLength());
            return;
        }
        if (from instanceof BytesWritable) {
            BytesWritable b = (BytesWritable) from;
            to.bytes(b.getBytes(), b.getLength());
            return;
        }

        super.convert(from, to);
    }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
    assertEquals(md5HashKey1, md5HashKey2);
  }

  @Test
  public void testMD5HashForBytesWritableKey() throws IOException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
  }

  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
View Full Code Here
767
768
769
770
771
772
773
774
775
776
777
778
779
    for(String outputFile : getAllFiles(dir)) {
      String name = (new File(outputFile)).getName();
      if(name.startsWith(prefix)) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
        LongWritable key = new LongWritable();
        BytesWritable value = new BytesWritable();
        while(reader.next(key, value)) {
          String body = new String(value.getBytes(), 0, value.getLength());
          if (bodies.contains(body)) {
            LOG.debug("Found event body: {}", body);
            bodies.remove(body);
            found++;
          }
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
278
        System.arraycopy(content, eol + 1, raw, 0, raw.length);
       
        // populate key and values with the header and raw content.
        Text keyText = (Text)key;
        keyText.set(header);
        BytesWritable valueBytes = (BytesWritable)value;
        valueBytes.set(raw, 0, raw.length);

        // TODO: It would be best to start at the end of the gzip read but
        // the bytes read in gzip don't match raw bytes in the file so we
        // overshoot the next header.  With this current method you get
        // some false positives but don't miss records.
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.BytesWritable

  • com.backtype.cascading.tap.PailTap$PailScheme
  • com.backtype.hadoop.pail.SequenceFileFormat$SequenceFilePailRecordReader
  • com.cloudera.sqoop.testutil.ManagerCompatTestCase
  • com.facebook.hive.orc.OrcTestUtils
  • com.facebook.hive.orc.TestOrcFile
  • com.facebook.hive.orc.WriterImpl$BinaryTreeWriter
  • com.facebook.presto.hive.util.SerDeUtils
  • com.pinterest.secor.tools.LogFileVerifier
  • edu.yale.cs.hadoopdb.connector.DBInputSplit
  • hipi.imagebundle.SeqImageBundle
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.