Package org.terrier.structures.CompressingMetaIndex

Examples of org.terrier.structures.CompressingMetaIndex.CompressingMetaIndexInputFormat


  {
    if (! validPlatform()) return;
    JobConf jc = HadoopPlugin.getJobFactory(this.getClass().getName()).newJob();
    HadoopUtility.toHConfiguration(index, jc);
    CompressingMetaIndexInputFormat.setStructure(jc, name);
    CompressingMetaIndexInputFormat information = new CompressingMetaIndexInputFormat();
    information.validateInput(jc);
    information.overrideDataFileBlockSize(blocksize);
    InputSplit[] splits = information.getSplits(jc, 2);
    Set<String> unseenDocnos = new HashSet<String>(Arrays.asList(docnos));
    int seenDocuments = 0;
    for(InputSplit split : splits)
    {
      RecordReader<IntWritable,Wrapper<String[]>> rr = information.getRecordReader(split, jc, null);
      IntWritable key = rr.createKey();
      Wrapper<String[]> value = rr.createValue();
      while(rr.next(key, value))
      {
        seenDocuments++;
View Full Code Here

TOP

Related Classes of org.terrier.structures.CompressingMetaIndex.CompressingMetaIndexInputFormat

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.