Examples of RecordReader

com.facebook.hive.orc.RecordReader
A row-by-row iterator for ORC files.
com.flaptor.util.sort.RecordReader
com.netflix.astyanax.util.RecordReader
@author elandau
org.apache.drill.exec.store.RecordReader
org.apache.hadoop.hive.ql.io.orc.RecordReader
A row-by-row iterator for ORC files.
org.apache.hadoop.mapred.RecordReader
RecordReader reads <key, value> pairs from an {@link InputSplit}.
RecordReader, typically, converts the byte-oriented view of the input, provided by the InputSplit, and presents a record-oriented view for the {@link Mapper} & {@link Reducer} tasks for processing. It thus assumes the responsibility of processing record boundaries and presenting the tasks with keys and values.
@see InputSplit @see InputFormat
org.apache.hadoop.mapreduce.RecordReader
The record reader breaks the data into key/value pairs for input to the {@link Mapper}. @param < KEYIN> @param < VALUEIN>
org.apache.hadoop.record.RecordReader
Front-end interface to deserializers. Also acts as a factory for deserializers. @author Milind Bhandarkar

Examples of com.facebook.hive.orc.RecordReader

            throw new IllegalArgumentException("DWRF does not support DATE type");
        }


        ReaderWriterProfiler.setProfilerOptions(configuration);


        RecordReader recordReader;
        try {
            FileSystem fileSystem = path.getFileSystem(configuration);
            Reader reader = OrcFile.createReader(fileSystem, path, new JobConf(configuration));
            boolean[] include = findIncludedColumns(reader.getTypes(), columns);
            recordReader = reader.rows(start, length, include);

View Full Code Here

Examples of com.flaptor.util.sort.RecordReader

        MergeSort.sort(beforeSort, afterSort, tcp, null);
        logger.info("Sorting categoryTokenCount.... done");


        logger.info("Folding categoryTokenCount.... ");
        // Now saves all this data to a FileCache        
        RecordReader tcrr= tcp.newRecordReader(afterSort);                
        TokenCounter catTc= new TokenCounter();
        TokenCounter nonCatTc= new TokenCounter();
        TokenCounterPersistence.TCRecord tcr= (TokenCounterPersistence.TCRecord) tcrr.readRecord();
        if (null==tcrr) return;
        catTc.update(tcr.getCatVal());
        nonCatTc.update(tcr.getNonCatVal());
        String prevToken= tcr.getToken();        
        while (null != (tcr= (TokenCounterPersistence.TCRecord) tcrr.readRecord())){
            if (tcr.getToken().equals(prevToken)){
                catTc.update(tcr.getCatVal());
                nonCatTc.update(tcr.getNonCatVal());
            } else {
                computeProbs(prevToken, catTc, nonCatTc, tokenProbabilities);

View Full Code Here

Examples of com.flaptor.util.sort.RecordReader

        WhoHasPersistence whp= new WhoHasPersistence();
        MergeSort.sort(beforeSort, afterSort, whp, null);
        logger.info("Sorting WhoHasPersistence... done");


        logger.info("Writing WhoHas Total to ... " + WHO_HAS_FILE_DONE);
        RecordReader whrr= whp.newRecordReader(afterSort); 
        WhoHasPersistence.WHRecord whr= (WhoHasPersistence.WHRecord) whrr.readRecord();        
        if (null==whr) return;
        
//        FileUtil.deleteFile(WHO_HAS_FILE_DONE);
        FileWriter whDone= new FileWriter(WHO_HAS_FILE_DONE);


        HashSet<String> urls= new HashSet<String>();
        for (String url: whr.getUrls()){
            urls.add(url);
        }
        String prevToken= whr.getToken();
        while (null != (whr= (WhoHasPersistence.WHRecord) whrr.readRecord())){
            if (whr.getToken().equals(prevToken)){
                for (String url: whr.getUrls()){
                    urls.add(url);
                }
            } else {

View Full Code Here

Examples of com.netflix.astyanax.util.RecordReader

                .append("1, eran, landau, 34, a\n")
                .append("2, netta, landau, 33, b\n")
                .append("3, arielle, landau, 6, c\n")
                .append("4, eyal, landau, 2, d\n");


        RecordReader reader = new CsvRecordReader(new StringReader(
                sb.toString()));
        RecordWriter writer = new ColumnarRecordWriter(keyspace,
                CF_USERS.getName());


        try {
            reader.start();
            writer.start();
            List<Pair<String, String>> record = null;
            while (null != (record = reader.next())) {
                writer.write(record);
            }
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
            Assert.fail();
        } catch (ConnectionException e) {
            LOG.error(e.getMessage(), e);
            Assert.fail();
        } finally {
            reader.shutdown();
            writer.shutdown();
        }


        try {
            Rows<Long, String> rows = keyspace.prepareQuery(CF_USERS)

View Full Code Here

Examples of org.apache.drill.exec.store.RecordReader

public class InfoSchemaBatchCreator implements BatchCreator<InfoSchemaSubScan>{
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaBatchCreator.class);


  @Override
  public RecordBatch getBatch(FragmentContext context, InfoSchemaSubScan config, List<RecordBatch> children) throws ExecutionSetupException {
    RecordReader rr = config.getTable().getRecordReader(context.getRootSchema());
    return new ScanBatch(config, context, Collections.singleton(rr).iterator());
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.orc.RecordReader

        this.counter = counter;
    }


    @Override
    public boolean readTo(T model) throws IOException {
        RecordReader reader = prepare();
        if (reader.hasNext() == false) {
            return false;
        }
        Object buf = reader.next(structBuffer);
        driver.set(model, buf);
        structBuffer = buf;


        advanceCounter((long) (fragmentSize * reader.getProgress()));
        return true;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.orc.RecordReader

        advanceCounter((long) (fragmentSize * reader.getProgress()));
        return true;
    }


    private RecordReader prepare() throws IOException {
        RecordReader reader = currentReader;
        if (reader == null) {
            if (LOG.isInfoEnabled()) {
                LOG.info(MessageFormat.format(
                        "Loading ORCFile metadata ({0}): {1}",
                        descriptor.getDataModelClass().getSimpleName(),

View Full Code Here

Examples of org.apache.hadoop.mapred.RecordReader

        InputSplit[] splits = format.getSplits(job, numSplits);


        // check each split
        BitSet bits = new BitSet(length);
        for (int j = 0; j < splits.length; j++) {
          RecordReader reader =
            format.getRecordReader(splits[j], job, Reporter.NULL);
          try {
            int count = 0;
            while (reader.next(key, value)) {
              assertFalse("Key in multiple partitions.", bits.get(key.getData()));
              bits.set(key.getData());
              count++;
            }
          } finally {
            reader.close();
          }
        }
        assertEquals("Some keys in no partition.", length, bits.cardinality());
      }

View Full Code Here

Examples of org.apache.hadoop.mapred.RecordReader

    pushProjectionsAndFilters(cloneJobConf, inputFormatClass, hsplit.getPath()
      .toString(), hsplit.getPath().toUri().getPath(), nonNative);


    InputFormat inputFormat = getInputFormatFromCache(inputFormatClass,
        cloneJobConf);
    RecordReader innerReader = inputFormat.getRecordReader(inputSplit,
        cloneJobConf, reporter);


    HiveRecordReader<K,V> rr = new HiveRecordReader(innerReader);
    rr.initIOContext(hsplit, job, inputFormatClass, innerReader);
    return rr;

View Full Code Here

Examples of org.apache.hadoop.mapred.RecordReader

    InputSplit[] splits = inputFormat.getSplits(jonconf, splitNumber);
    assertEquals("splits length should be " + splitNumber, splits.length, splitNumber);
    int readCount = 0;
    for (int i = 0; i < splits.length; i++) {
      int previousReadCount = readCount;
      RecordReader rr = inputFormat.getRecordReader(splits[i], jonconf, Reporter.NULL);
      Object key = rr.createKey();
      Object value = rr.createValue();
      while (rr.next(key, value)) {
        readCount++;
      }
      System.out.println("The " + i + "th split read "
          + (readCount - previousReadCount));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.