Examples of OutputLogFilter

org.apache.hadoop.mapred.OutputLogFilter
This class filters log files from directory given It doesnt accept paths having _logs. This can be used to list paths of output directory as follows: Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir, new OutputLogFilter())); @deprecated Use {@link org.apache.hadoop.mapred.Utils.OutputFileUtils.OutputLogFilter} instead.
org.apache.mahout.clustering.kmeans.OutputLogFilter

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter

    throws IOException, InstantiationException, IllegalAccessException {


    Map<Integer, List<VectorWritable>> repPoints = RepresentativePointsMapper.getRepresentativePoints(conf, stateIn);
    Map<Integer, WeightedVectorWritable> mostDistantPoints = new HashMap<Integer, WeightedVectorWritable>();
    FileSystem fs = FileSystem.get(clusteredPointsIn.toUri(), conf);
    FileStatus[] status = fs.listStatus(clusteredPointsIn, new OutputLogFilter());
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      try {
        IntWritable key = (IntWritable) reader.getKeyClass().asSubclass(Writable.class).newInstance();
        WeightedVectorWritable vw = reader.getValueClass().asSubclass(WeightedVectorWritable.class).newInstance();

View Full Code Here

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter

                                            double alpha,
                                            int k) throws IOException {
    DirichletState state = DirichletDriver.createState(modelDistribution, k, alpha);
    Path path = new Path(statePath);
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    FileStatus[] status = fs.listStatus(path, new OutputLogFilter());
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      try {
        Writable key = new Text();
        DirichletCluster cluster = new DirichletCluster();

View Full Code Here

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter


  protected static List<DirichletCluster> loadClusters(Configuration conf, Path stateIn) {
    List<DirichletCluster> clusters = new ArrayList<DirichletCluster>();
    try {
      FileSystem fs = FileSystem.get(stateIn.toUri(), conf);
      FileStatus[] status = fs.listStatus(stateIn, new OutputLogFilter());
      for (FileStatus s : status) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
        try {
          Text key = new Text();
          DirichletCluster cluster = new DirichletCluster();

View Full Code Here

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter

  protected static List<Cluster> readClusters(Path clustersIn)
    throws IOException, InstantiationException, IllegalAccessException {
    List<Cluster> clusters = new ArrayList<Cluster>();
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(clustersIn.toUri(), conf);
    FileStatus[] status = fs.listStatus(clustersIn, new OutputLogFilter());
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      try {
        Writable key = new Text();
        Writable value = reader.getValueClass().asSubclass(Writable.class).newInstance();

View Full Code Here

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter

      throws InstantiationException, IllegalAccessException, IOException {
    CanopyClusterer clusterer = new CanopyClusterer(measure, t1, t2);
    Collection<Canopy> canopies = new ArrayList<Canopy>();
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    FileStatus[] status = fs.listStatus(input, new OutputLogFilter());
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      try {
        Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
        VectorWritable vw = reader.getValueClass().asSubclass(VectorWritable.class).newInstance();

View Full Code Here

Examples of org.apache.mahout.clustering.kmeans.OutputLogFilter

    CanopyClusterer clusterer = new CanopyClusterer(measure, t1, t2);


    Collection<Canopy> clusters = new ArrayList<Canopy>();
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(canopies.toUri(), conf);
    FileStatus[] status = fs.listStatus(canopies, new OutputLogFilter());
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      try {
        Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
        Canopy value = reader.getValueClass().asSubclass(Canopy.class).newInstance();
        while (reader.next(key, value)) {
          clusters.add(value);
          value = reader.getValueClass().asSubclass(Canopy.class).newInstance();
        }
      } finally {
        reader.close();
      }
    }
    // iterate over all points, assigning each to the closest canopy and outputing that clustering
    fs = FileSystem.get(points.toUri(), conf);
    status = fs.listStatus(points, new OutputLogFilter());
    Path outPath = new Path(output, DEFAULT_CLUSTERED_POINTS_DIRECTORY);
    int part = 0;
    for (FileStatus s : status) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
      SequenceFile.Writer writer = new SequenceFile.Writer(fs,

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.