Package org.apache.mahout.classifier.df.data

Examples of org.apache.mahout.classifier.df.data.Data

241
242
243
244
245
246
247
248
249
250
  }
 
  protected static Data loadData(Configuration conf, Path dataPath, Dataset dataset) throws IOException {
    log.info("Loading the data...");
    FileSystem fs = dataPath.getFileSystem(conf);
    Data data = DataLoader.loadData(dataset, fs, dataPath);
    log.info("Data Loaded");
   
    return data;
  }
View Full Code Here
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    Node childNode;
    if (data.getDataset().isNumerical(best.getAttr())) {
      boolean[] temp = null;

      Data loSubset = data.subset(Condition.lesser(best.getAttr(), best.getSplit()));
      Data hiSubset = data.subset(Condition.greaterOrEquals(best.getAttr(), best.getSplit()));

      if (loSubset.isEmpty() || hiSubset.isEmpty()) {
        // the selected attribute did not change the data, avoid using it in the child notes
        selected[best.getAttr()] = true;
      } else {
        // the data changed, so we can unselect all previousely selected NUMERICAL attributes
        temp = selected;
        selected = cloneCategoricalAttributes(data.getDataset(), selected);
      }

      // size of the subset is less than the minSpitNum
      if (loSubset.size() < minSplitNum || hiSubset.size() < minSplitNum) {
        // branch is not split
        double label;
        if (data.getDataset().isNumerical(data.getDataset().getLabelId())) {
          label = sum / data.size();
        } else {
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
  @Override
  protected void cleanup(Context context) throws IOException, InterruptedException {
    // prepare the data
    log.debug("partition: {} numInstances: {}", partition, instances.size());
   
    Data data = new Data(getDataset(), instances);
    Bagging bagging = new Bagging(getTreeBuilder(), data);
   
    TreeID key = new TreeID();
   
    log.debug("Building {} trees", nbTrees);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
    super.setup(context);
   
    Configuration conf = context.getConfiguration();
   
    log.info("Loading the data...");
    Data data = loadData(conf, getDataset());
    log.info("Data loaded : {} instances", data.size());
   
    bagging = new Bagging(getTreeBuilder(), data);
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    double hy = entropy(data); // H(Y)
    double hyx = 0.0; // H(Y|X)
    double invDataSize = 1.0 / data.size();
   
    for (double value : values) {
      Data subset = data.subset(Condition.equals(attr, value));
      hyx += subset.size() * invDataSize * entropy(subset);
    }
   
    return hy - hyx;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
  double numericalIg(Data data, int attr, double split) {
    double hy = entropy(data);
    double invDataSize = 1.0 / data.size();
   
    // LO subset
    Data subset = data.subset(Condition.lesser(attr, split));
    hy -= subset.size() * invDataSize * entropy(subset);
   
    // HI subset
    subset = data.subset(Condition.greaterOrEquals(attr, split));
    hy -= subset.size() * invDataSize * entropy(subset);
   
    return hy;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
   * Builds one tree
   */
  public Node build(Random rng) {
    log.debug("Bagging...");
    Arrays.fill(sampled, false);
    Data bag = data.bagging(rng, sampled);
   
    log.debug("Building...");
    return treeBuilder.build(rng, bag);
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
  public void testComputeSplit() throws Exception {
    IgSplit ref = new DefaultIgSplit();
    IgSplit opt = new OptIgSplit();

    Random rng = RandomUtils.getRandom();
    Data data = Utils.randomData(rng, NUM_ATTRIBUTES, false, NUM_INSTANCES);

    for (int nloop = 0; nloop < 100; nloop++) {
      int attr = rng.nextInt(data.getDataset().nbAttributes());
      // System.out.println("IsNumerical: " + data.dataset.isNumerical(attr));

      Split expected = ref.computeSplit(data, attr);
      Split actual = opt.computeSplit(data, attr);
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    String[] source = Utils.double2String(dData);
    String descriptor = "N N N N N N N N L";

    Dataset dataset = DataLoader.generateDataset(descriptor, false, source);
    Data data = DataLoader.loadData(dataset, source);
    TreeBuilder builder = new DecisionTreeBuilder();
    builder.build(rng, data);

    // regression
    dataset = DataLoader.generateDataset(descriptor, true, source);
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    Node childNode;
    if (data.getDataset().isNumerical(best.getAttr())) {
      boolean[] temp = null;

      Data loSubset = data.subset(Condition.lesser(best.getAttr(), best.getSplit()));
      Data hiSubset = data.subset(Condition.greaterOrEquals(best.getAttr(), best.getSplit()));

      if (loSubset.isEmpty() || hiSubset.isEmpty()) {
        // the selected attribute did not change the data, avoid using it in the child notes
        selected[best.getAttr()] = true;
      } else {
        // the data changed, so we can unselect all previousely selected NUMERICAL attributes
        temp = selected;
        selected = cloneCategoricalAttributes(data.getDataset(), selected);
      }

      // size of the subset is less than the minSpitNum
      if (loSubset.size() < minSplitNum || hiSubset.size() < minSplitNum) {
        // branch is not split
        double label;
        if (data.getDataset().isNumerical(data.getDataset().getLabelId())) {
          label = sum / data.size();
        } else {
View Full Code Here
TOP

Related Classes of org.apache.mahout.classifier.df.data.Data

  • org.apache.mahout.classifier.df.Bagging
  • org.apache.mahout.classifier.df.BreimanExample
  • org.apache.mahout.classifier.df.builder.DecisionTreeBuilder
  • org.apache.mahout.classifier.df.builder.DefaultTreeBuilder
  • org.apache.mahout.classifier.df.builder.InfiniteRecursionTest
  • org.apache.mahout.classifier.df.DecisionForestTest
  • org.apache.mahout.classifier.df.mapreduce.BuildForest
  • org.apache.mahout.classifier.df.mapreduce.inmem.InMemMapper
  • org.apache.mahout.classifier.df.mapreduce.partial.Step1Mapper
  • org.apache.mahout.classifier.df.split.DefaultIgSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.