Package org.apache.mahout.df.data

Examples of org.apache.mahout.df.data.Data

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
  @Override
  public void close() throws IOException {
    // prepare the data
    log.debug("partition: " + partition + "numInstances: " + instances.size());
   
    Data data = new Data(getDataset(), instances);
    Bagging bagging = new Bagging(getTreeBuilder(), data);

    TreeID key = new TreeID();

    log.debug("Building " + nbTrees + " trees");
    SingleTreePredictions callback = null;
    int[] predictions = null;
    for (int treeId = 0; treeId < nbTrees; treeId++) {
      log.debug("Building tree N° : " + treeId);
      if (isOobEstimate() && !isNoOutput()) {
        callback = new SingleTreePredictions(data.size());
        predictions = callback.getPredictions();
      }

      Node tree = bagging.build(treeId, rng, callback);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    double hy = entropy(data); // H(Y)
    double hyx = 0.0; // H(Y|X)
    double invDataSize = 1.0 / data.size();

    for (double value : values) {
      Data subset = data.subset(Condition.equals(attr, value));
      hyx += subset.size() * invDataSize * entropy(subset);
    }

    return hy - hyx;
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  protected double numericalIg(Data data, int attr, double split) {
    double hy = entropy(data);
    double invDataSize = 1.0 / data.size();

    // LO subset
    Data subset = data.subset(Condition.lesser(attr, split));
    hy -= subset.size() * invDataSize * entropy(subset);

    // HI subset
    subset = data.subset(Condition.greaterOrEquals(attr, split));
    hy -= subset.size() * invDataSize * entropy(subset);

    return hy;
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
  @Override
  protected void cleanup(Context context) throws IOException, InterruptedException {
    // prepare the data
    log.debug("partition: " + partition + "numInstances: " + instances.size());

    Data data = new Data(getDataset(), instances);
    Bagging bagging = new Bagging(getTreeBuilder(), data);

    TreeID key = new TreeID();

    log.debug("Building " + nbTrees + " trees");
    SingleTreePredictions callback = null;
    int[] predictions = null;
    for (int treeId = 0; treeId < nbTrees; treeId++) {
      log.debug("Building tree N° : " + treeId);
      if (isOobEstimate() && !isNoOutput()) {
        callback = new SingleTreePredictions(data.size());
        predictions = callback.getPredictions();
      }

      Node tree = bagging.build(treeId, rng, callback);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    IgSplit ref = new DefaultIgSplit();
    IgSplit opt = new OptIgSplit();

    Random rng = RandomUtils.getRandom();
    Data data = Utils.randomData(rng, nbAttributes, numInstances);

    for (int nloop = 0; nloop < n; nloop++) {
      int attr = rng.nextInt(data.getDataset().nbAttributes());
      // System.out.println("IsNumerical: " + data.dataset.isNumerical(attr));

      Split expected = ref.computeSplit(data, attr);
      Split actual = opt.computeSplit(data, attr);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
    // store the data into a file
    String[] sData = Utils.double2String(source);
    Path dataPath = Utils.writeDataToTestFile(sData);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);

    JobConf jobConf = new JobConf();
    jobConf.setNumMapTasks(numMaps);

    // prepare a custom TreeBuilder that will classify each
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    // store the data into a file
    String[] sData = Utils.double2String(source);
    Path dataPath = Utils.writeDataToTestFile(sData);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);

    Configuration conf = new Configuration();
    Step0JobTest.setMaxSplitSize(conf, dataPath, numMaps);

    // prepare a custom TreeBuilder that will classify each
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
    for (int nloop = 0; nloop < n; nloop++) {
      int nbAttributes = rng.nextInt(maxNbAttributes) + 1;

      // generate a small data, only to get the dataset
      Data data = Utils.randomData(rng, nbAttributes, 1);
      if (data.getDataset().nbAttributes() == 0)
        continue;

      int m = rng.nextInt(data.getDataset().nbAttributes()) + 1;

      int[] attrs = DefaultTreeBuilder.randomAttributes(data.getDataset(), rng, m);

      assertEquals(m, attrs.length);

      for (int index = 0; index < m; index++) {
        int attr = attrs[index];
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    // all the vectors have the same label (0)
    double[][] temp = Utils.randomDoublesWithSameLabel(rng, descriptor, 100, 0);
    String[] sData = Utils.double2String(temp);
    Dataset dataset = DataLoader.generateDataset(descriptor, sData);
    Data data = DataLoader.loadData(dataset, sData);
    DefaultIgSplit iG = new DefaultIgSplit();

    double expected = 0.0 - 1.0 * Math.log(1.0) / Math.log(2.0);
    assertEquals(expected, iG.entropy(data));

View Full Code Here
223
224
225
226
227
228
229
230
231
232
  }
 
  protected static Data loadData(Configuration conf, Path dataPath, Dataset dataset) throws IOException {
    log.info("Loading the data...");
    FileSystem fs = dataPath.getFileSystem(conf);
    Data data = DataLoader.loadData(dataset, fs, dataPath);
    log.info("Data Loaded");
   
    return data;
  }
View Full Code Here
TOP

Related Classes of org.apache.mahout.df.data.Data

  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.mahout.common.FileLineIterator
  • org.apache.mahout.common.iterator.FileLineIterator
  • org.apache.mahout.df.Bagging
  • org.apache.mahout.df.BreimanExample
  • org.apache.mahout.df.builder.DefaultTreeBuilder
  • org.apache.mahout.df.builder.DefaultTreeBuilderTest
  • org.apache.mahout.df.builder.InfiniteRecursionTest
  • org.apache.mahout.df.mapred.BuildForest
  • org.apache.mahout.df.mapred.partial.PartitionBugTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.