Package de.lmu.ifi.dbs.elki.datasource.bundle

Examples of de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
        String l2 = objects.data(o2, lblcol).toString();
        return l1.compareToIgnoreCase(l2);
      }
    });

    MultipleObjectsBundle bundle = new MultipleObjectsBundle();
    for(int j = 0; j < objects.metaLength(); j++) {
      // Reorder column accordingly
      List<?> in = objects.getColumn(j);
      List<Object> data = new ArrayList<Object>(size);
      for(int i = 0; i < size; i++) {
        data.add(in.get(offsets[i]));
      }
      bundle.appendColumn(objects.meta(j), data);
    }
    return bundle;
  }
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
   * for each object) and indexes and fires a deletion event.
   */
  @Override
  public MultipleObjectsBundle delete(DBIDs ids) {
    // Prepare bundle to return
    MultipleObjectsBundle bundle = new MultipleObjectsBundle();
    for(Relation<?> relation : relations) {
      ArrayList<Object> data = new ArrayList<Object>(ids.size());
      for(DBID id : ids) {
        data.add(relation.get(id));
      }
      bundle.appendColumn(relation.getDataTypeInformation(), data);
    }
    // remove from db
    for(DBID id : ids) {
      doDelete(id);
    }
View Full Code Here
50
51
52
53
54
55
56
 
  @Override
  public MultipleObjectsBundle loadData() {
    // Return an empty bundle
    // TODO: add some dummy column, such as DBIDs?
    return new MultipleObjectsBundle();
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    List<MultipleObjectsBundle> bundles = new ArrayList<MultipleObjectsBundle>(sources.size());
    for(DatabaseConnection dbc : sources) {
      bundles.add(dbc.loadData());
    }

    MultipleObjectsBundle first = bundles.get(0);
    Map<String, Integer> labelmap = new HashMap<String, Integer>(first.dataLength());
    // Process first bundle
    {
      // Identify a label column
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < first.metaLength(); i++) {
          if(TypeUtil.GUESSED_LABEL.isAssignableFromType(first.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        throw new AbortException("No label column found in first source, cannot join (do you want to use " + ExternalIDJoinDatabaseConnection.class.getSimpleName() + " instead?)");
      }
      for(int i = 0; i < first.dataLength(); i++) {
        Object data = first.data(i, lblcol);
        if(data == null) {
          logger.warning("Object without label encountered.");
          continue;
        }
        if(data instanceof String) {
          Integer old = labelmap.put((String) data, i);
          if(old != null) {
            logger.warning("Duplicate label encountered: " + data + " in rows " + old + " and " + i);
          }
        }
        else if(data instanceof LabelList) {
          for(String lbl : (LabelList) data) {
            Integer old = labelmap.put(lbl, i);
            if(old != null) {
              logger.warning("Duplicate label encountered: " + lbl + " in rows " + old + " and " + i);
            }
          }
        }
        else {
          String lbl = data.toString();
          Integer old = labelmap.put(lbl, i);
          if(old != null) {
            logger.warning("Duplicate label encountered: " + lbl + " in rows " + old + " and " + i);
          }
        }
      }
    }
    // Process additional columns
    for(int c = 1; c < sources.size(); c++) {
      MultipleObjectsBundle cur = bundles.get(c);
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < cur.metaLength(); i++) {
          if(TypeUtil.GUESSED_LABEL.isAssignableFromType(cur.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        throw new AbortException("No label column found in source " + (c + 1) + ", cannot join (do you want to use " + ExternalIDJoinDatabaseConnection.class.getSimpleName() + " instead?)");
      }
      // Destination columns
      List<ArrayList<Object>> dcol = new ArrayList<ArrayList<Object>>(cur.metaLength());
      for(int i = 0; i < cur.metaLength(); i++) {
        // Skip the label columns
        if(i == lblcol) {
          dcol.add(null);
          continue;
        }
        ArrayList<Object> newcol = new ArrayList<Object>(first.dataLength());
        // Pre-fill with nulls.
        for(int j = 0; j < first.dataLength(); j++) {
          newcol.add(null);
        }
        first.appendColumn(cur.meta(i), newcol);
        dcol.add(newcol);
      }
      for(int i = 0; i < cur.dataLength(); i++) {
        Object data = cur.data(i, lblcol);
        if(data == null) {
          logger.warning("Object without label encountered.");
          continue;
        }
        Integer row = null;
        if(data instanceof String) {
          row = labelmap.get(data);
        }
        else if(data instanceof LabelList) {
          for(String lbl : (LabelList) data) {
            row = labelmap.get(lbl);
            if(row != null) {
              break;
            }
          }
        }
        else {
          row = labelmap.get(data.toString());
        }
        if(row == null) {
          logger.warning("Label not found for join: " + data + " in row " + i);
          continue;
        }
        for(int d = 0; d < cur.metaLength(); d++) {
          if(d == lblcol) {
            continue;
          }
          List<Object> col = dcol.get(d);
          assert (col != null);
          col.set(row, cur.data(i, d));
        }
      }
    }
    for(int i = 0; i < first.dataLength(); i++) {
      for(int d = 0; d < first.metaLength(); d++) {
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  public MultipleObjectsBundle loadData() {
    // Run parser
    if(logger.isDebugging()) {
      logger.debugFine("Invoking parsers.");
    }
    MultipleObjectsBundle parsingResult = parser.parse(in);

    // normalize objects and transform labels
    if(logger.isDebugging()) {
      logger.debugFine("Invoking filters.");
    }
    MultipleObjectsBundle objects = invokeFilters(parsingResult);
    return objects;
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
      int[] dimsize = new int[names.size()];
      processColumnTypes(names, types, targ, elkitypes, dimsize);

      // Prepare bundle:
      // This is a bit complicated to produce vector fields.
      MultipleObjectsBundle bundle = new MultipleObjectsBundle();
      StreamTokenizer tokenizer = makeArffTokenizer(br);

      int state = 0;

      nextToken(tokenizer);
      while(tokenizer.ttype != StreamTokenizer.TT_EOF) {
        // Parse instance
        if(tokenizer.ttype == StreamTokenizer.TT_EOL) {
          // ignore empty lines
        }
        else if(tokenizer.ttype != '{') {
          if(state == 0) {
            setupBundleHeaders(names, targ, elkitypes, dimsize, bundle, false);
            state = 1; // dense
          }
          if(state != 1) {
            throw new AbortException("Mixing dense and sparse vectors is currently not allowed.");
          }
          // Load a dense instance
          bundle.appendSimple(loadDenseInstance(tokenizer, dimsize, elkitypes, bundle.metaLength()));
        }
        else {
          if(state == 0) {
            setupBundleHeaders(names, targ, elkitypes, dimsize, bundle, true);
            state = 2; // dense
          }
          if(state != 2) {
            throw new AbortException("Mixing dense and sparse vectors is currently not allowed.");
          }
          bundle.appendSimple(loadSparseInstance(tokenizer, targ, dimsize, elkitypes, bundle.metaLength()));
        }
        if(tokenizer.ttype != StreamTokenizer.TT_EOF) {
          nextToken(tokenizer);
        }
      }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    List<MultipleObjectsBundle> bundles = new ArrayList<MultipleObjectsBundle>(sources.size());
    for(DatabaseConnection dbc : sources) {
      bundles.add(dbc.loadData());
    }

    MultipleObjectsBundle first = bundles.get(0);
    Map<ExternalID, Integer> labelmap = new HashMap<ExternalID, Integer>(first.dataLength());
    // Process first bundle
    {
      // Identify a label column
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < first.metaLength(); i++) {
          if(TypeUtil.EXTERNALID.isAssignableFromType(first.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        throw new AbortException("No external ID column found in primary source.");
      }
      for(int i = 0; i < first.dataLength(); i++) {
        ExternalID data = (ExternalID) first.data(i, lblcol);
        if(data == null) {
          logger.debug("Object without ID encountered.");
          continue;
        }
        Integer old = labelmap.put(data, i);
        if(old != null) {
          logger.debug("Duplicate id encountered: " + data + " in rows " + old + " and " + i);
        }
      }
    }
    // Process additional columns
    for(int c = 1; c < sources.size(); c++) {
      MultipleObjectsBundle cur = bundles.get(c);
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < cur.metaLength(); i++) {
          if(TypeUtil.EXTERNALID.isAssignableFromType(cur.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        StringBuffer buf = new StringBuffer();
        for(int i = 0; i < cur.metaLength(); i++) {
          if(buf.length() > 0) {
            buf.append(",");
          }
          buf.append(cur.meta(i));
        }
        throw new AbortException("No external ID column found in source " + (c + 1) + " to join with. Got: " + buf.toString());
      }
      // Destination columns
      List<ArrayList<Object>> dcol = new ArrayList<ArrayList<Object>>(cur.metaLength());
      for(int i = 0; i < cur.metaLength(); i++) {
        // Skip the label columns
        if(i == lblcol) {
          dcol.add(null);
          continue;
        }
        ArrayList<Object> newcol = new ArrayList<Object>(first.dataLength());
        // Pre-fill with nulls.
        for(int j = 0; j < first.dataLength(); j++) {
          newcol.add(null);
        }
        first.appendColumn(cur.meta(i), newcol);
        dcol.add(newcol);
      }
      for(int i = 0; i < cur.dataLength(); i++) {
        ExternalID data = (ExternalID) cur.data(i, lblcol);
        if(data == null) {
          logger.warning("Object without label encountered.");
          continue;
        }
        Integer row = labelmap.get(data);
        if(row == null) {
          logger.debug("ID not found for join: " + data + " in row " + i);
          continue;
        }
        for(int d = 0; d < cur.metaLength(); d++) {
          if(d == lblcol) {
            continue;
          }
          List<Object> col = dcol.get(d);
          assert (col != null);
          col.set(row, cur.data(i, d));
        }
      }
    }
    for(int i = 0; i < first.dataLength(); i++) {
      for(int d = 0; d < first.metaLength(); d++) {
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
      final int temp = offsets[j];
      offsets[j] = offsets[i - 1];
      offsets[i - 1] = temp;
    }

    MultipleObjectsBundle bundle = new MultipleObjectsBundle();
    for(int j = 0; j < objects.metaLength(); j++) {
      // Reorder column accordingly
      List<?> in = objects.getColumn(j);
      List<Object> data = new ArrayList<Object>(size);
      for(int i = 0; i < size; i++) {
        data.add(in.get(offsets[i]));
      }
      bundle.appendColumn(objects.meta(j), data);
    }
    return bundle;
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    List<MultipleObjectsBundle> bundles = new ArrayList<MultipleObjectsBundle>(sources.size());
    for(DatabaseConnection dbc : sources) {
      bundles.add(dbc.loadData());
    }

    MultipleObjectsBundle first = bundles.get(0);
    Map<ExternalID, Integer> labelmap = new HashMap<ExternalID, Integer>(first.dataLength());
    // Process first bundle
    {
      // Identify a label column
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < first.metaLength(); i++) {
          if(TypeUtil.EXTERNALID.isAssignableFromType(first.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        throw new AbortException("No external ID column found in primary source.");
      }
      for(int i = 0; i < first.dataLength(); i++) {
        ExternalID data = (ExternalID) first.data(i, lblcol);
        if(data == null) {
          logger.debug("Object without ID encountered.");
          continue;
        }
        Integer old = labelmap.put(data, i);
        if(old != null) {
          logger.debug("Duplicate id encountered: " + data + " in rows " + old + " and " + i);
        }
      }
    }
    // Process additional columns
    for(int c = 1; c < sources.size(); c++) {
      MultipleObjectsBundle cur = bundles.get(c);
      final int lblcol;
      {
        int lblc = -1;
        for(int i = 0; i < cur.metaLength(); i++) {
          if(TypeUtil.EXTERNALID.isAssignableFromType(cur.meta(i))) {
            lblc = i;
            break;
          }
        }
        lblcol = lblc; // make static
      }
      if(lblcol == -1) {
        StringBuffer buf = new StringBuffer();
        for(int i = 0; i < cur.metaLength(); i++) {
          if(buf.length() > 0) {
            buf.append(",");
          }
          buf.append(cur.meta(i));
        }
        throw new AbortException("No external ID column found in source " + (c + 1) + " to join with. Got: " + buf.toString());
      }
      // Destination columns
      List<ArrayList<Object>> dcol = new ArrayList<ArrayList<Object>>(cur.metaLength());
      for(int i = 0; i < cur.metaLength(); i++) {
        // Skip the label columns
        if(i == lblcol) {
          dcol.add(null);
          continue;
        }
        ArrayList<Object> newcol = new ArrayList<Object>(first.dataLength());
        // Pre-fill with nulls.
        for(int j = 0; j < first.dataLength(); j++) {
          newcol.add(null);
        }
        first.appendColumn(cur.meta(i), newcol);
        dcol.add(newcol);
      }
      for(int i = 0; i < cur.dataLength(); i++) {
        ExternalID data = (ExternalID) cur.data(i, lblcol);
        if(data == null) {
          logger.warning("Object without label encountered.");
          continue;
        }
        Integer row = labelmap.get(data);
        if(row == null) {
          logger.debug("ID not found for join: " + data + " in row " + i);
          continue;
        }
        for(int d = 0; d < cur.metaLength(); d++) {
          if(d == lblcol) {
            continue;
          }
          List<Object> col = dcol.get(d);
          assert (col != null);
          col.set(row, cur.data(i, d));
        }
      }
    }
    for(int i = 0; i < first.dataLength(); i++) {
      for(int d = 0; d < first.metaLength(); d++) {
View Full Code Here
50
51
52
53
54
55
56
 
  @Override
  public MultipleObjectsBundle loadData() {
    // Return an empty bundle
    // TODO: add some dummy column, such as DBIDs?
    return new MultipleObjectsBundle();
  }
View Full Code Here
TOP

Related Classes of de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle

  • de.lmu.ifi.dbs.elki.application.GeneratorXMLSpec
  • de.lmu.ifi.dbs.elki.data.synthetic.bymodel.GeneratorMain
  • de.lmu.ifi.dbs.elki.database.HashmapDatabase
  • de.lmu.ifi.dbs.elki.database.StaticArrayDatabase
  • de.lmu.ifi.dbs.elki.datasource.AbstractDatabaseConnection
  • de.lmu.ifi.dbs.elki.datasource.bundle.BundleStreamSource.Event
  • de.lmu.ifi.dbs.elki.datasource.ConcatenateFilesDatabaseConnection
  • de.lmu.ifi.dbs.elki.datasource.EmptyDatabaseConnection
  • de.lmu.ifi.dbs.elki.datasource.ExternalIDJoinDatabaseConnection
  • de.lmu.ifi.dbs.elki.datasource.filter.AbstractConversionFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.