Package org.terrier.structures.indexing.singlepass.hadoop

Examples of org.terrier.structures.indexing.singlepass.hadoop.MapData


    final int thisPartition = TaskAttemptID.forName(jc.get("mapred.task.id")).getTaskID().getId();
    final SplitEmittedTerm.SETPartitioner partitionChecker = new SplitEmittedTerm.SETPartitioner();
    partitionChecker.configure(jc);
   
   
    MapData tempHRD;
    for (FileStatus file : files)
    {
      //logger.info("Run data file "+ file.getPath().toString()+" has length "+Files.length(file.getPath().toString()));
      runDataIn = new DataInputStream(Files.openFileStream(file.getPath().toString()));
      tempHRD = new MapData(runDataIn);
      //check to see if this file contaned our split information
      if (mutipleIndices && partitionChecker.calculatePartition(tempHRD.getSplitnum(), jc.getNumReduceTasks()) != thisPartition)
        continue;
     
      mapTaskIDs.add(tempHRD.getMap());
      runData.add(tempHRD);
      runDataIn.close();
    }
    // Sort by splitnum
    Collections.sort(runData);
View Full Code Here

TOP

Related Classes of org.terrier.structures.indexing.singlepass.hadoop.MapData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.