Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.Text$Comparator

56
57
58
59
60
61
62
63
64
65
66
  public String read() throws IOException {
    if (reader == null) {
      reader = getInput();
    }

    Text k = new Text();
    Text v = new Text();
    reader.next(k, v);
    byte[] value = v.getBytes();
    return value != null && value.length > 0 ? new String(value) : null;
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
      readerHelper = new ReaderHelper<LineReader, byte[]>(getInput(), getInputContext(), getSplit(), getCodec()) {
        @Override
        protected LineReader createReader(InputStream inputStream) throws IOException {
          LineReader lineReader = new LineReader(inputStream, delimiter);
          if (getContext().getStart() > 0) {
            processReadCount(lineReader.readLine(new Text()));
          }
          return lineReader;
        }

        @Override
        protected byte[] doRead(LineReader delegate) throws IOException {
          Text text = new Text();
          if (!getInputContext().isEndReached()) {
            processReadCount(delegate.readLine(text));
          }
          return text.getBytes();
        }
      };
      readerHelper.init();
    }
    byte[] value = readerHelper.read();
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  @Override
  public synchronized void write(String entity) throws IOException {
    if (holder == null) {
      holder = getOutput();
    }
    holder.getWriter().append(NULL_KEY, new Text(entity.getBytes()));

    setWritePosition(getPosition(holder.getWriter()));

    OutputContext context = getOutputContext();
    if (context.getRolloverState()) {
View Full Code Here
297
298
299
300
301
302
303
      }
      host = addr.getAddress().getHostAddress();
    } else {
      host = addr.getHostName().toLowerCase();
    }
    return new Text(host + ":" + addr.getPort());
  }
View Full Code Here
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
   
    if (id.getUser() == null) {
      throw new InvalidToken("Token with no owner");
    }
    String owner = id.getUser().getUserName();
    Text renewer = id.getRenewer();
    KerberosName cancelerKrbName = new KerberosName(canceller);
    String cancelerShortName = cancelerKrbName.getShortName();
    if (!canceller.equals(owner)
        && (renewer == null || "".equals(renewer.toString()) || !cancelerShortName
            .equals(renewer.toString()))) {
      throw new AccessControlException(canceller
          + " is not authorized to cancel the token");
    }
    DelegationTokenInformation info = null;
    info = currentTokens.remove(id);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
    protected ScannerBase addColFamsAndQuals(ScannerBase scan, AccumuloParameters parameters) {
        String[] colFamsAndQuals = parameters.getColumns();
        for(String pair : colFamsAndQuals) {
            String[] colFamAndQual = colon.split(pair);
            if(colFamAndQual.length == 1){
                scan.fetchColumnFamily(new Text(colFamAndQual[0]));
            } else if(colFamAndQual.length == 2){
                scan.fetchColumn(new Text(colFamAndQual[0]), new Text(colFamAndQual[1]));
            } else {
                throw new IllegalArgumentException("malformed colFam entry: " + pair);
            }
        }
        if(colFamsAndQuals.length == 0 && log.isDebugEnabled())
View Full Code Here
23
24
25
26
27
28
29
30
31
32
33
34
35
public class TestDelimitedLineRecordReader {
  @Test
  public void testGetColumnFixedSizeCols() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd,cdd,efd,ghd,ijd,kld"), val); // checking
                                // parsed text
                                // value
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
  }

  @Test
  public void testGetColumnVariableSize() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abcde,cedd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abcde,cedd,efd,ghd,ijd,kld"), val);
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  @Test
  public void testGetColumnForDelimiterSizeGreaterThanOne()
      throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd::cdd::efd::ghd::ijd::kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, "::");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd::cdd::efd::ghd::ijd::kld"), val);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
  }

  @Test
  public void testGetColumnStartingWithDelimiter() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text(",abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ghd"), key);
    assertEquals(new Text(",abd,cdd,efd,ghd,ijd,kld"), val);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.Text$Comparator

  • azkaban.common.web.JsonSequenceFileViewer
  • bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest
  • bbejeck.mapred.coocurrance.StripesOccurrenceMapperTest
  • bbejeck.mapred.coocurrance.WordPair
  • cellmate.accumulo.writer.dbtransformers.AccumuloDBItemTransformers
  • co.nubetech.hiho.dedup.DelimitedLineRecordReader
  • co.nubetech.hiho.dedup.HihoTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.