Package org.apache.mahout.df.data

Examples of org.apache.mahout.df.data.Dataset

88
89
90
91
92
93
94
95
96
97
98
    // prepare the data
    String descriptor = Utils.randomDescriptor(rng, nbAttributes);
    double[][] source = Utils.randomDoubles(rng, descriptor, nbInstances);
    String[] sData = Utils.double2String(source);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    String[][] splits = Utils.splitData(sData, nbMappers);

    MockTreeBuilder treeBuilder = new MockTreeBuilder();

    LongWritable key = new LongWritable();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    }

    // store the data into a file
    String[] sData = Utils.double2String(source);
    Path dataPath = Utils.writeDataToTestFile(sData);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);

    Configuration conf = new Configuration();
    Step0JobTest.setMaxSplitSize(conf, dataPath, numMaps);
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  private DecisionForest buildForest() throws IOException, ClassNotFoundException, InterruptedException {
    DefaultTreeBuilder treeBuilder = new DefaultTreeBuilder();
    treeBuilder.setM(m);

   Dataset dataset = Dataset.load(getConf(), datasetPath);

    ForestPredictions callback = (isOob) ? new ForestPredictions(dataset
        .nbInstances(), dataset.nblabels()) : null;

    Builder forestBuilder;

    if (isPartial) {
      log.info("Partial Mapred implementation");
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
      return -1;
    }
   
    // load the data
    FileSystem fs = dataPath.getFileSystem(new Configuration());
    Dataset dataset = Dataset.load(getConf(), datasetPath);
    Data data = DataLoader.loadData(dataset, fs, dataPath);

    // take m to be the first integer less than log2(M) + 1, where M is the
    // number of inputs
    int m = (int) Math.floor(Maths.log(2, data.getDataset().nbAttributes()) + 1);
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
171
172
173
  private DecisionForest buildForest() throws IOException {
    DefaultTreeBuilder treeBuilder = new DefaultTreeBuilder();
    treeBuilder.setM(m);

    Dataset dataset = Dataset.load(getConf(), datasetPath);

    ForestPredictions callback = (isOob) ? new ForestPredictions(dataset
        .nbInstances(), dataset.nblabels()) : null;

    Builder forestBuilder;

    if (isPartial) {
      log.info("Partial Mapred implementation");
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    if (files == null || files.length < 2) {
      throw new IllegalArgumentException("missing paths from the DistributedCache");
    }

    Path datasetPath = new Path(files[0].getPath());
    Dataset dataset = Dataset.load(conf, datasetPath);

    int numMaps = Builder.getNumMaps(conf);
    int p = conf.getInt("mapred.task.partition", -1);

    // total number of trees in the forest
View Full Code Here
110
111
112
113
114
115
116
117
118
119
    String descriptor = DescriptorUtils.generateDescriptor(description);

    Path fPath = validateOutput(filePath);
   
    log.info("generating the dataset...");
    Dataset dataset = generateDataset(descriptor, dataPath);

    log.info("storing the dataset description");
    storeWritable(new Configuration(), fPath, dataset);
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    if (files == null || files.length < 2) {
      throw new IllegalArgumentException("missing paths from the DistributedCache");
    }

    Dataset dataset;
    try {
      Path datasetPath = new Path(files[0].getPath());
      dataset = Dataset.load(job, datasetPath);
    } catch (IOException e) {
      throw new IllegalStateException("Exception while loading the dataset : ", e);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    }

    // store the data into a file
    String[] sData = Utils.double2String(source);
    Path dataPath = Utils.writeDataToTestFile(sData);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);

    JobConf jobConf = new JobConf();
    jobConf.setNumMapTasks(numMaps);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    }

    // store the data into a file
    String[] sData = Utils.double2String(source);
    Path dataPath = Utils.writeDataToTestFile(sData);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);

    Configuration conf = new Configuration();
    Step0JobTest.setMaxSplitSize(conf, dataPath, numMaps);
View Full Code Here
TOP

Related Classes of org.apache.mahout.df.data.Dataset

  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.mahout.df.BreimanExample
  • org.apache.mahout.df.builder.InfiniteRecursionTest
  • org.apache.mahout.df.mapred.BuildForest
  • org.apache.mahout.df.mapred.partial.PartitionBugTest
  • org.apache.mahout.df.mapred.partial.Step0JobTest
  • org.apache.mahout.df.mapred.partial.Step1MapperTest
  • org.apache.mahout.df.mapred.partial.Step2Mapper
  • org.apache.mahout.df.mapred.partial.Step2MapperTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.