Examples of org.apache.hadoop.io.Text

org.apache.hadoop.io.Text
This class stores text using standard UTF8 encoding. It provides methods to serialize, deserialize, and compare texts at byte level. The type of length is integer and is serialized using zero-compressed format.
In addition, it provides methods for string traversal without converting the byte array to a string.
Also includes utilities for serializing/deserialing a string, coding/decoding a string, checking if a byte array contains valid UTF8 code, calculating the length of an encoded string.

    Assert.assertEquals(globalExpected, actual);
  }


  @Test(expected=IOException.class)
  public void testInvalidReadClass() throws IOException {
    wal.writeEntry(new WALEntry<Text>(new Text(""), 1));
    wal.close();
    new WAL<IntWritable>(dataDir, IntWritable.class);
  }

View Full Code Here

  }


  @Test(expected=NullPointerException.class)
  public void testCloseSingle() throws IOException {
    wal.close();
    wal.writeEntry(new WALEntry<Text>(new Text(""), 1));
  }

View Full Code Here


  @SuppressWarnings("unchecked")
  @Test(expected=NullPointerException.class)
  public void testCloseList() throws IOException {
    wal.close();
    wal.writeEntries(Lists.newArrayList(new WALEntry<Text>(new Text(""), 1)));
  }

View Full Code Here

  }
  private MapWritable toMapWritable(Map<String, String> map) {
    MapWritable result = new MapWritable();
    if(map != null) {
      for(Map.Entry<String, String> entry : map.entrySet()) {
        result.put(new Text(entry.getKey()),new Text(entry.getValue()));
      }
    }
    return result;
  }

View Full Code Here

    SequenceFile.Writer writer
      = new SequenceFile.Writer(FileSystem.get(uri, c), c,
                                new Path(uri.toString()),
                                LongWritable.class, Text.class);
    final LongWritable key = new LongWritable();
    final Text val = new Text();
    for (int i = 0; i < COUNT; ++i) {
      key.set(i);
      val.set(Integer.toString(i));
      writer.append(key, val);
    }
    writer.close();
  }

View Full Code Here

  private static Token<AuthenticationTokenIdentifier> getAuthToken(Configuration conf, User user)
      throws IOException, InterruptedException {
    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
    try {
      String clusterId = ZKClusterId.readClusterIdZNode(zkw);
      return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getUGI().getTokens());
    } catch (KeeperException e) {
      throw new IOException(e);
    } finally {
      zkw.close();
    }

View Full Code Here

            }


            Path dst = makeRelative(root, stat.getPath());
            src_writer.append(new LongWritable(stat.isDir()? 0: stat.getLen()),
                new FilePair(stat, dst));
            dst_writer.append(new Text(dst.toString()),
                new Text(stat.getPath().toString()));
          }
        }
      }
    } finally {
      checkAndClose(src_writer);

View Full Code Here

      SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
        new Text.Comparator(), Text.class, conf);
      sorter.sort(file, sorted);
      in = new SequenceFile.Reader(fs, sorted, conf);


      Text prevdst = null, curdst = new Text();
      Text prevsrc = null, cursrc = new Text(); 
      for(; in.next(curdst, cursrc); ) {
        if (prevdst != null && curdst.equals(prevdst)) {
          throw new DuplicationException(
            "Invalid input, there are duplicated files in the sources: "
            + prevsrc + ", " + cursrc);
        }
        prevdst = curdst;
        curdst = new Text();
        prevsrc = cursrc;
        cursrc = new Text();
      }
    }
    finally {
      checkAndClose(in);
    }

View Full Code Here

        return;
      }


      if (destFileSys.exists(absdst) && !overwrite
          && !needsUpdate(srcstat, destFileSys.getFileStatus(absdst))) {
        outc.collect(null, new Text("SKIP: " + srcstat.getPath()));
        ++skipcount;
        reporter.incrCounter(Counter.SKIP, 1);
        updateStatus(reporter);
        return;
      }

View Full Code Here

        ++failcount;
        reporter.incrCounter(Counter.FAIL, 1);
        updateStatus(reporter);
        final String sfailure = "FAIL " + dstpath + " : " +
                          StringUtils.stringifyException(e);
        out.collect(null, new Text(sfailure));
        LOG.info(sfailure);
        try {
          for (int i = 0; i < 3; ++i) {
            try {
              if (destFileSys.delete(dstpath))

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Text

azkaban.common.web.JsonSequenceFileViewer

bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest

bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest

bbejeck.mapred.coocurrance.StripesOccurrenceMapperTest

bbejeck.mapred.coocurrance.WordPair

cellmate.accumulo.writer.dbtransformers.AccumuloDBItemTransformers

co.nubetech.hiho.dedup.DelimitedLineRecordReader

co.nubetech.hiho.dedup.HihoTuple

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.