Package org.apache.mahout.df.data

Examples of org.apache.mahout.df.data.Dataset

130
131
132
133
134
135
136
137
138
139
140
    // write the data to a file
    Path dataPath = Utils.writeDataToTestFile(sData);
   
    // prepare a data converter
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    DataConverter converter = new DataConverter(dataset);
   
    JobConf job = new JobConf();
    job.setNumMapTasks(numMaps);
    FileInputFormat.setInputPaths(job, dataPath);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    // prepare the data
    String descriptor = Utils.randomDescriptor(rng, nbAttributes);
    double[][] source = Utils.randomDoubles(rng, descriptor, nbInstances);
    String[] sData = Utils.double2String(source);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    String[][] splits = Utils.splitData(sData, nbMappers);

    // prepare first step output
    TreeID[] keys = new TreeID[nbTrees];
    Node[] trees = new Node[nbTrees];
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    // prepare the data
    String descriptor = Utils.randomDescriptor(rng, nbAttributes);
    double[][] source = Utils.randomDoubles(rng, descriptor, nbInstances);
    String[] sData = Utils.double2String(source);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    String[][] splits = Utils.splitData(sData, nbMappers);

    MockTreeBuilder treeBuilder = new MockTreeBuilder();

    LongWritable key = new LongWritable();
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
      log.error("No Decision Forest found!");
      return;
    }

    // load the dataset
    Dataset dataset = Dataset.load(getConf(), datasetPath);
    DataConverter converter = new DataConverter(dataset);

    log.info("Sequential classification...");
    long time = System.currentTimeMillis();

    Random rng = RandomUtils.getRandom();
    ResultAnalyzer analyzer = analyze ? new ResultAnalyzer(Arrays.asList(dataset.labels()), "unknown") : null;

    if (dataFS.getFileStatus(dataPath).isDir()) {
      //the input is a directory of files
      testDirectory(outputPath, converter, forest, dataset, analyzer, rng);
    }  else {
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
 
  private DecisionForest buildForest() throws IOException, ClassNotFoundException, InterruptedException {
    DefaultTreeBuilder treeBuilder = new DefaultTreeBuilder();
    treeBuilder.setM(m);
   
    Dataset dataset = Dataset.load(getConf(), datasetPath);
   
    ForestPredictions callback = isOob ? new ForestPredictions(dataset.nbInstances(), dataset.nblabels())
        : null;
   
    Builder forestBuilder;
   
    if (isPartial) {
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
      return -1;
    }
   
    // load the data
    FileSystem fs = dataPath.getFileSystem(new Configuration());
    Dataset dataset = Dataset.load(getConf(), datasetPath);
    Data data = DataLoader.loadData(dataset, fs, dataPath);
   
    // take m to be the first integer less than log2(M) + 1, where M is the
    // number of inputs
    int m = (int) Math.floor(Maths.log(2, data.getDataset().nbAttributes()) + 1);
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
    }

    DefaultTreeBuilder treeBuilder = new DefaultTreeBuilder();
    treeBuilder.setM(m);
   
    Dataset dataset = Dataset.load(getConf(), datasetPath);
   
    ForestPredictions callback = isOob ? new ForestPredictions(dataset.nbInstances(), dataset.nblabels())
        : null;
   
    Builder forestBuilder;
   
    if (isPartial) {
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    // prepare the data
    String descriptor = Utils.randomDescriptor(rng, NUM_ATTRIBUTES);
    double[][] source = Utils.randomDoubles(rng, descriptor, NUM_INSTANCES);
    String[] sData = Utils.double2String(source);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    String[][] splits = Utils.splitData(sData, NUM_MAPPERS);

    // prepare first step output
    TreeID[] keys = new TreeID[NUM_TREES];
    Node[] trees = new Node[NUM_TREES];
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    // prepare the data
    String descriptor = Utils.randomDescriptor(rng, NUM_ATTRIBUTES);
    double[][] source = Utils.randomDoubles(rng, descriptor, NUM_INSTANCES);
    String[] sData = Utils.double2String(source);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    String[][] splits = Utils.splitData(sData, NUM_MAPPERS);

    MockTreeBuilder treeBuilder = new MockTreeBuilder();

    LongWritable key = new LongWritable();
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
    int label = Utils.findLabel(descriptor);

    // all the vectors have the same label (0)
    double[][] temp = Utils.randomDoublesWithSameLabel(rng, descriptor, 100, 0);
    String[] sData = Utils.double2String(temp);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);
    DefaultIgSplit iG = new DefaultIgSplit();

    double expected = 0.0 - 1.0 * Math.log(1.0) / Math.log(2.0);
    assertEquals(expected, iG.entropy(data), EPSILON);
View Full Code Here
TOP

Related Classes of org.apache.mahout.df.data.Dataset

  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.mahout.df.BreimanExample
  • org.apache.mahout.df.builder.InfiniteRecursionTest
  • org.apache.mahout.df.mapred.BuildForest
  • org.apache.mahout.df.mapred.partial.PartitionBugTest
  • org.apache.mahout.df.mapred.partial.Step0JobTest
  • org.apache.mahout.df.mapred.partial.Step1MapperTest
  • org.apache.mahout.df.mapred.partial.Step2Mapper
  • org.apache.mahout.df.mapred.partial.Step2MapperTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.