Package ivory.core.data.document

Examples of ivory.core.data.document.WeightedIntDocVector

275
276
277
278
279
280
281
282
283
284
285
  }
 
  private WeightedIntDocVector findIntDocVector(int docno, int startFrom, FileSystem fs, String dir) {
    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();
    try {
      FileStatus[] paths = fs.listStatus(new Path(dir));
      System.err.println("length " + paths.length);
      for (FileStatus path : paths) {
        if (!path.getPath().getName().contains("part"))  continue;
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs,
        new Path(galagoIndex + "/wt-int-doc-vectors/part-00002"), fs.getConf());
    reader.next(key, value);
    verifyIntDocVector(galagoIntDocVector1, value);
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs,
        new Path(opennlpIndex + "/wt-int-doc-vectors/part-00002"), fs.getConf());
    reader.next(key, value);
    System.out.println("opennlpInt1\n"+value);
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
        Reporter reporter) throws IOException {
      Long time = System.currentTimeMillis();
      for(int i=0;i<vectors.size();i++){
        IntWritable sampleDocno = (IntWritable)vectors.get(i).getLeftElement();

        WeightedIntDocVector fromSample = (WeightedIntDocVector)vectors.get(i).getRightElement();
        float cs = CLIRUtils.cosine(docvector.getWeightedTerms(), fromSample.getWeightedTerms());
        if(cs >= threshold){
          output.collect(new IntWritable(sampleDocno.get()), new PairOfFloatInt(cs,docno.get()));
        }
      }
      sLogger.info("Finished in "+(System.currentTimeMillis()-time));
View Full Code Here
157
158
159
160
161
162
163
164
165
        for(MapIF.Entry e : vectorWeights.entrySet()){
          float score = vectorWeights.get(e.getKey());
          vectorWeights.put(e.getKey(), score/sum2);
        }
      }
      WeightedIntDocVector weightedVector = new WeightedIntDocVector (docLen, vectorWeights);
      output.collect(mDocno, weightedVector);
      reporter.incrCounter(Docs.Total, 1);
    }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
    public void map(IntWritable docno, WeightedIntDocVector docvector,
        OutputCollector<IntWritable, PairOfFloatInt> output, Reporter reporter) throws IOException {
      for (int i = 0; i < vectors.size(); i++) {
        IntWritable sampleDocno = (IntWritable) vectors.get(i).getLeftElement();

        WeightedIntDocVector fromSample = (WeightedIntDocVector) vectors.get(i).getRightElement();
        float cs = CLIRUtils.cosine(docvector.getWeightedTerms(), fromSample.getWeightedTerms());

        if (cs >= threshold) {
          output.collect(new IntWritable(sampleDocno.get()), new PairOfFloatInt(cs, docno.get()));
        }
      }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
    public void map(IntWritable docno, WeightedIntDocVector docvector,
        OutputCollector<IntWritable, PairOfFloatInt> output, Reporter reporter) throws IOException {
      for (int i = 0; i < vectors.size(); i++) {
        IntWritable sampleDocno = (IntWritable) vectors.get(i).getLeftElement();

        WeightedIntDocVector fromSample = (WeightedIntDocVector) vectors.get(i).getRightElement();
        float cs = CLIRUtils.cosine(docvector.getWeightedTerms(), fromSample.getWeightedTerms());

        if (cs >= threshold) {
          output.collect(new IntWritable(sampleDocno.get()), new PairOfFloatInt(cs, docno.get()));
        }
      }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(enwikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(enIntDocVector1, value);
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(dewikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(deIntDocVector1, value);
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();

    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(galagoIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("galagoInt1\n"+key+";"+value);
View Full Code Here
TOP

Related Classes of ivory.core.data.document.WeightedIntDocVector

  • edu.umd.cloud9.io.map.HMapIFW
  • ivory.core.preprocess.BuildWeightedIntDocVectors$MyMapper
  • ivory.integration.wikipedia.VerifyWikipediaProcessingCrosslingual
  • ivory.integration.wikipedia.VerifyWikipediaProcessingMonolingual
  • ivory.lsh.eval.BruteForcePwsim$MyMapperDocVectors
  • ivory.lsh.eval.BruteForcePwsim$MyMapperWeightedIntDocVectors
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.