Examples of org.apache.hadoop.io.Text

org.apache.hadoop.io.Text
This class stores text using standard UTF8 encoding. It provides methods to serialize, deserialize, and compare texts at byte level. The type of length is integer and is serialized using zero-compressed format.
In addition, it provides methods for string traversal without converting the byte array to a string.
Also includes utilities for serializing/deserialing a string, coding/decoding a string, checking if a byte array contains valid UTF8 code, calculating the length of an encoded string.


  public void readFields(DataInput in) throws IOException {
    int version = in.readByte();
    if (version < 6) {
      if (version <= 2) {
        Text t = new Text();
        t.readFields(in);
        this.name = t.getBytes();
//        if(KeyValue.getFamilyDelimiterIndex(this.name, 0, this.name.length)
//            > 0) {
//          this.name = stripColon(this.name);
//        }
      } else {

View Full Code Here

    LOG.debug("Input folder location: " + inputFolderPath);


    List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
    Path[] inputFiles = new Path[splits.size()];


    Text key = new Text();
    for (int i = 0; i < inputFiles.length; i++) {
      List<Path> files = splits.get(i);
      inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
      SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
        Text.class, NullWritable.class);
      LOG.debug("Input split: " + i);
      try {
        for (Path file: files) {
          LOG.debug(file.toString());
          key.set(file.toString());
          writer.append(key, NullWritable.get());
        }
      } finally {
        writer.close();
      }

View Full Code Here

    InetAddress ia = HBaseServer.getRemoteIp();
    ServerName rs = this.serverManager.regionServerStartup(ia, port,
      serverStartCode, serverCurrentTime);
    // Send back some config info
    MapWritable mw = createConfigurationSubset();
    mw.put(new Text(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER),
      new Text(rs.getHostname()));
    return mw;
  }

View Full Code Here

    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    return addConfig(mw, "fs.default.name");
  }


  private MapWritable addConfig(final MapWritable mw, final String key) {
    mw.put(new Text(key), new Text(this.conf.get(key)));
    return mw;
  }

View Full Code Here

  }
  private MapWritable toMapWritable(Map<String, String> map) {
    MapWritable result = new MapWritable();
    if(map != null) {
      for(Map.Entry<String, String> entry : map.entrySet()) {
        result.put(new Text(entry.getKey()),new Text(entry.getValue()));
      }
    }
    return result;
  }

View Full Code Here

    wal.close();
    wal = new WAL<Text>(dataDir, Text.class, 0L, 0L, 0L, 1L);
    long seqid = 0;
    List<String> expected = strings(100);
    for(String s : expected) {
      wal.writeEntry(new WALEntry<Text>(new Text(s), ++seqid));
      Thread.sleep(1);
      wal.writeSequenceID(seqid);
      Thread.sleep(1);
    }
    wal.writeSequenceID(Long.MAX_VALUE);

View Full Code Here

  @Test
  public void testBasicReplay() throws IOException {
    long seqid = 0;
    List<String> expected = strings(100);
    for(String s : expected) {
      wal.writeEntry(new WALEntry<Text>(new Text(s), ++seqid));
    }
    wal.close();
    wal = new WAL<Text>(dataDir, Text.class);
    WALReplayResult<Text> result = wal.replay();
    Assert.assertEquals(100, result.getSequenceID());

View Full Code Here

  @Test
  public void testReplayAtOffset() throws IOException {
    long seqid = 0;
    List<String> expected = strings(100);
    for(String s : expected) {
      wal.writeEntry(new WALEntry<Text>(new Text(s), ++seqid));
    }
    wal.writeSequenceID(50);
    expected.remove(50);
    wal.close();
    wal = new WAL<Text>(dataDir, Text.class);

View Full Code Here

  @Test
  public void testReplayNone() throws IOException {
    long seqid = 0;
    List<String> expected = strings(100);
    for(String s : expected) {
      wal.writeEntry(new WALEntry<Text>(new Text(s), ++seqid));
      wal.writeSequenceID(seqid);
    }
    wal.close();
    wal = new WAL<Text>(dataDir, Text.class);
    WALReplayResult<Text> result = wal.replay();

View Full Code Here

            startLatch.countDown();
            startLatch.await();
            // half batch, half do not
            if(id % 2 == 0) {
              for(String s : expected) {
                wal.writeEntry(new WALEntry<Text>(new Text(s), seqid.incrementAndGet()));
              }
            } else {
              List<WALEntry<Text>> batch = Lists.newArrayList();
              for(String s : expected) {
                batch.add(new WALEntry<Text>(new Text(s), seqid.incrementAndGet()));
              }
              wal.writeEntries(batch);
            }
          } catch (Exception e) {
            logger.warn("Error doing appends", e);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Text

azkaban.common.web.JsonSequenceFileViewer

bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest

bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest

bbejeck.mapred.coocurrance.StripesOccurrenceMapperTest

bbejeck.mapred.coocurrance.WordPair

cellmate.accumulo.writer.dbtransformers.AccumuloDBItemTransformers

co.nubetech.hiho.dedup.DelimitedLineRecordReader

co.nubetech.hiho.dedup.HihoTuple

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.