Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.Path.toUri()

29
30
31
32
33
34
35
36
37
38
39
public class FileInfosDatasetTest extends MahoutTestCase {

  public void testRanges() throws IOException {
    Path inpath = new Path("target/test-classes/wdbc");
    FileSystem fs = FileSystem.get(inpath.toUri(), new Configuration());
   
    DataSet dataset = FileInfoParser.parseFile(fs, inpath);
    DataSet.initialize(dataset);

    DataLine dl = new DataLine();
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
      // random descriptions
      Object[][] descriptions = randomDescriptions(descriptors);

      // random dataset
      Path inpath = new Path("input");
      FileSystem fs = FileSystem.get(inpath.toUri(), new Configuration());
      if (fs.exists(inpath)) {
        fs.delete(inpath, true);
      }

      randomDataset(fs, inpath, descriptors, descriptions);
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
  private HadoopUtil() { }
 
  public static void overwriteOutput(String output) throws IOException {
    Configuration conf = new JobConf(KMeansDriver.class);
    Path outPath = new Path(output);
    FileSystem fs = FileSystem.get(outPath.toUri(), conf);
    if (fs.exists(outPath)) {
      log.warn("Deleting {}", outPath);
      fs.delete(outPath, true);
    }
    log.warn("Creating dir {}", outPath);
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
   * @throws RuntimeException
   *           if the data could not be loaded
   */
  private static Data loadData(Configuration conf, Dataset dataset) throws IOException {
    Path dataPath = Builder.getDistributedCacheFile(conf, 1);
    FileSystem fs = FileSystem.get(dataPath.toUri(), conf);
    return DataLoader.loadData(dataset, fs, dataPath);
  }
 
  @Override
  protected void setup(Context context) throws IOException, InterruptedException {
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    SequenceFile.Writer inputVectorPathWriter = new SequenceFile.Writer(fs,
            conf, inputVectorPath, NullWritable.class, VectorWritable.class);
    VectorWritable inputVW = new VectorWritable(v);
    inputVectorPathWriter.append(NullWritable.get(), inputVW);
    inputVectorPathWriter.close();
    URI ivpURI = inputVectorPath.toUri();
    DistributedCache.setCacheFiles(new URI[] {ivpURI}, conf);
    fs.deleteOnExit(inputVectorPath);

    conf.set(INPUT_VECTOR, ivpURI.toString());
    conf.setBoolean(IS_SPARSE_OUTPUT, !(v instanceof DenseVector));
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
    for (int k = 0; k < points.size(); k++) {
      System.out.println("testKFuzzyKMeansMRJob k= " + k);
      // pick k initial cluster centers at random
      JobConf job = new JobConf(FuzzyKMeansDriver.class);
      Path path = new Path("testdata/clusters");
      FileSystem fs = FileSystem.get(path.toUri(), job);
      if (fs.exists(path)) {
        fs.delete(path, true);
      }
     
      testData = new File("testdata/clusters");
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
       
      }
      writer.close();
     
      Path outPath = new Path("output");
      fs = FileSystem.get(outPath.toUri(), conf);
      if (fs.exists(outPath)) {
        fs.delete(outPath, true);
      }
      fs.mkdirs(outPath);
      // now run the Job
View Full Code Here
358
359
360
361
362
363
364
365
366
367
368
    for (int k = 1; k < points.size(); k++) {
      System.out.println("testKMeansMRJob k= " + k);
      // pick k initial cluster centers at random
      JobConf job = new JobConf(KMeansDriver.class);
      Path path = new Path("testdata/clusters/part-00000");
      FileSystem fs = FileSystem.get(path.toUri(), job);
      SequenceFile.Writer writer = new SequenceFile.Writer(fs, job, path, Text.class, Cluster.class);
     
      for (int i = 0; i < k + 1; i++) {
        Vector vec = points.get(i).get();
       
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
   * @throws RuntimeException
   *           if the data could not be loaded
   */
  private static Data loadData(JobConf conf, Dataset dataset) throws IOException {
    Path dataPath = Builder.getDistributedCacheFile(conf, 1);
    FileSystem fs = FileSystem.get(dataPath.toUri(), conf);
    return DataLoader.loadData(dataset, fs, dataPath);
  }
 
  @Override
  public void configure(JobConf conf) {
View Full Code Here
345
346
347
348
349
350
351
352
353
354
355
    // now run the Canopy Driver
    CanopyDriver.runJob("testdata", "output/canopies", ManhattanDistanceMeasure.class.getName(), 3.1, 2.1);
   
    // verify output from sequence file
    Path path = new Path("output/canopies/part-00000");
    FileSystem fs = FileSystem.get(path.toUri(), job);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, job);
    Text key = new Text();
    Canopy canopy = new Canopy();
    assertTrue("more to come", reader.next(key, canopy));
    assertEquals("1st key", "C0", key.toString());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.