Package ivory.core.data.document

Examples of ivory.core.data.document.WeightedIntDocVector

267
268
269
270
271
272
273
274
275
276
277
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(opennlpIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("opennlpInt1\n"+key+";"+value);
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(enwikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(enIntDocVector1, value);
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(dewikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(deIntDocVector1, value);
View Full Code Here
171
172
173
174
175
176
177
178
179
        for (MapIF.Entry e : vectorWeights.entrySet()) {
          float score = vectorWeights.get(e.getKey());
          vectorWeights.put(e.getKey(), score / sum2);
        }
      }
      WeightedIntDocVector weightedVector = new WeightedIntDocVector(docLen, vectorWeights);
      output.collect(mDocno, weightedVector);
      reporter.incrCounter(Docs.Total, 1);
    }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(galagoIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("galagoInt1\n"+key+";"+value);
View Full Code Here
264
265
266
267
268
269
270
271
272
273
274
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(opennlpIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("opennlpInt1\n"+key+";"+value);
View Full Code Here
167
168
169
170
171
172
173
174
175
        for (MapIF.Entry e : vectorWeights.entrySet()) {
          float score = vectorWeights.get(e.getKey());
          vectorWeights.put(e.getKey(), score / sum2);
        }
      }
      WeightedIntDocVector weightedVector = new WeightedIntDocVector(docLen, vectorWeights);
      output.collect(mDocno, weightedVector);
      reporter.incrCounter(Docs.Total, 1);
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
  public void verifyIntDocVectorsEn() throws Exception {
    System.out.println("verifyIntDocVectorsEn");
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    WeightedIntDocVector value = findIntDocVector(enIntDocVector1Id, 2, fs, enwikiEn + "/wt-int-doc-vectors");
    System.out.println("enIntDocVector1\n"+value);
    verifyIntDocVector(enIntDocVector1, value);

    value = findIntDocVector(enIntDocVector2Id, 11, fs, enwikiEn + "/wt-int-doc-vectors");
    System.out.println("enIntDocVector2\n"+value);
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
  public void verifyIntDocVectorsDe() throws Exception {
    System.out.println("verifyIntDocVectorsDe");
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    WeightedIntDocVector value = findIntDocVector(deIntDocVector1Id, 2, fs, dewikiEn + "/wt-int-doc-vectors");
    System.out.println("deIntDocVector1\n"+value);
    verifyIntDocVector(deIntDocVector1, value);

    value = findIntDocVector(deIntDocVector2Id, 11, fs, dewikiEn + "/wt-int-doc-vectors");
    System.out.println("deIntDocVector2\n"+value);
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
  }
 
  private WeightedIntDocVector findIntDocVector(int docno, int startFrom, FileSystem fs, String dir) {
    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();
    try {
      FileStatus[] paths = fs.listStatus(new Path(dir));
      System.err.println("length " + paths.length);
      for (FileStatus path : paths) {
        if (!path.getPath().getName().contains("part"))  continue;
View Full Code Here
TOP

Related Classes of ivory.core.data.document.WeightedIntDocVector

  • edu.umd.cloud9.io.map.HMapIFW
  • ivory.core.preprocess.BuildWeightedIntDocVectors$MyMapper
  • ivory.integration.wikipedia.VerifyWikipediaProcessingCrosslingual
  • ivory.integration.wikipedia.VerifyWikipediaProcessingMonolingual
  • ivory.lsh.eval.BruteForcePwsim$MyMapperDocVectors
  • ivory.lsh.eval.BruteForcePwsim$MyMapperWeightedIntDocVectors
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.