Examples of org.apache.hadoop.io.Text

org.apache.hadoop.io.Text
This class stores text using standard UTF8 encoding. It provides methods to serialize, deserialize, and compare texts at byte level. The type of length is integer and is serialized using zero-compressed format.
In addition, it provides methods for string traversal without converting the byte array to a string.
Also includes utilities for serializing/deserialing a string, coding/decoding a string, checking if a byte array contains valid UTF8 code, calculating the length of an encoded string.

  public String read() throws IOException {
    if (reader == null) {
      reader = getInput();
    }


    Text k = new Text();
    Text v = new Text();
    reader.next(k, v);
    byte[] value = v.getBytes();
    return value != null && value.length > 0 ? new String(value) : null;
  }

View Full Code Here

      readerHelper = new ReaderHelper<LineReader, byte[]>(getInput(), getInputContext(), getSplit(), getCodec()) {
        @Override
        protected LineReader createReader(InputStream inputStream) throws IOException {
          LineReader lineReader = new LineReader(inputStream, delimiter);
          if (getContext().getStart() > 0) {
            processReadCount(lineReader.readLine(new Text()));
          }
          return lineReader;
        }


        @Override
        protected byte[] doRead(LineReader delegate) throws IOException {
          Text text = new Text();
          if (!getInputContext().isEndReached()) {
            processReadCount(delegate.readLine(text));
          }
          return text.getBytes();
        }
      };
      readerHelper.init();
    }
    byte[] value = readerHelper.read();

View Full Code Here

  @Override
  public synchronized void write(String entity) throws IOException {
    if (holder == null) {
      holder = getOutput();
    }
    holder.getWriter().append(NULL_KEY, new Text(entity.getBytes()));


    setWritePosition(getPosition(holder.getWriter()));


    OutputContext context = getOutputContext();
    if (context.getRolloverState()) {

View Full Code Here

      }
      host = addr.getAddress().getHostAddress();
    } else {
      host = addr.getHostName().toLowerCase();
    }
    return new Text(host + ":" + addr.getPort());
  }

View Full Code Here

    
    if (id.getUser() == null) {
      throw new InvalidToken("Token with no owner");
    }
    String owner = id.getUser().getUserName();
    Text renewer = id.getRenewer();
    KerberosName cancelerKrbName = new KerberosName(canceller);
    String cancelerShortName = cancelerKrbName.getShortName();
    if (!canceller.equals(owner)
        && (renewer == null || "".equals(renewer.toString()) || !cancelerShortName
            .equals(renewer.toString()))) {
      throw new AccessControlException(canceller
          + " is not authorized to cancel the token");
    }
    DelegationTokenInformation info = null;
    info = currentTokens.remove(id);

View Full Code Here

    protected ScannerBase addColFamsAndQuals(ScannerBase scan, AccumuloParameters parameters) {
        String[] colFamsAndQuals = parameters.getColumns();
        for(String pair : colFamsAndQuals) {
            String[] colFamAndQual = colon.split(pair);
            if(colFamAndQual.length == 1){
                scan.fetchColumnFamily(new Text(colFamAndQual[0]));
            } else if(colFamAndQual.length == 2){
                scan.fetchColumn(new Text(colFamAndQual[0]), new Text(colFamAndQual[1]));
            } else {
                throw new IllegalArgumentException("malformed colFam entry: " + pair);
            }
        }
        if(colFamsAndQuals.length == 0 && log.isDebugEnabled())

View Full Code Here


public class TestDelimitedLineRecordReader {
  @Test
  public void testGetColumnFixedSizeCols() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd,cdd,efd,ghd,ijd,kld"), val); // checking
                                // parsed text
                                // value
  }

View Full Code Here

  }


  @Test
  public void testGetColumnVariableSize() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abcde,cedd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abcde,cedd,efd,ghd,ijd,kld"), val);
  }

View Full Code Here


  @Test
  public void testGetColumnForDelimiterSizeGreaterThanOne()
      throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd::cdd::efd::ghd::ijd::kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, "::");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd::cdd::efd::ghd::ijd::kld"), val);
  }

View Full Code Here

  }


  @Test
  public void testGetColumnStartingWithDelimiter() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text(",abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ghd"), key);
    assertEquals(new Text(",abd,cdd,efd,ghd,ijd,kld"), val);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Text

azkaban.common.web.JsonSequenceFileViewer

bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapperTest

bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest

bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest

bbejeck.mapred.coocurrance.StripesOccurrenceMapperTest

bbejeck.mapred.coocurrance.WordPair

cellmate.accumulo.writer.dbtransformers.AccumuloDBItemTransformers

co.nubetech.hiho.dedup.DelimitedLineRecordReader

co.nubetech.hiho.dedup.HihoTuple

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.