Package net.yacy.kelondro.order

Examples of net.yacy.kelondro.order.Bitfield

146
147
148
149
150
151
152
153
154
155
156
        this.appdate = entry.getColLong(5);
        this.profileHandle = (entry.empty(6)) ? null : entry.getColASCII(6).trim();
        this.depth = (int) entry.getColLong(7);
        this.anchors = (int) entry.getColLong(8);
        this.forkfactor = (int) entry.getColLong(9);
        this.flags = new Bitfield(entry.getColBytes(10, true));
        //this.loaddate = entry.getColLong(12);
        //this.lastmodified = entry.getColLong(13);
        this.size = entry.getColLong(14);
        this.statusMessage        = "loaded(kelondroRow.Entry)";
        this.initialHash   = this.url.hashCode();
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
        this.entry.setCol(col_size, Integer.parseInt(prop.getProperty("size", "0")));
        this.entry.setCol(col_wc, Integer.parseInt(prop.getProperty("wc", "0")));
        final String dt = prop.getProperty("dt", "t");
        this.entry.setCol(col_dt, dt.length() > 0 ? new byte[]{(byte) dt.charAt(0)} : new byte[]{(byte) 't'});
        final String flags = prop.getProperty("flags", "AAAAAA");
        this.entry.setCol(col_flags, (flags.length() > 6) ? QueryParams.empty_constraint.bytes() : (new Bitfield(4, flags)).bytes());
        this.entry.setCol(col_lang, UTF8.getBytes(prop.getProperty("lang", "uk")));
        this.entry.setCol(col_llocal, Integer.parseInt(prop.getProperty("llocal", "0")));
        this.entry.setCol(col_lother, Integer.parseInt(prop.getProperty("lother", "0")));
        this.entry.setCol(col_limage, Integer.parseInt(prop.getProperty("limage", "0")));
        this.entry.setCol(col_laudio, Integer.parseInt(prop.getProperty("laudio", "0")));
View Full Code Here
433
434
435
436
437
438
439
    public int size() {
        return (int) this.entry.getColLong(col_size);
    }

    public Bitfield flags() {
        return new Bitfield(this.entry.getColBytes(col_flags, true));
    }
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
            if (post.containsKey("urllist")) {
                if (keystring.length() == 0 || !ByteBuffer.equals(Word.word2hash(keystring), keyhash)) {
                    prop.put("keystring", "<not possible to compute word from hash>");
                }
                final Bitfield flags = compileFlags(post);
                final int count = (post.get("lines", "all").equals("all")) ? -1 : post.getInt("lines", -1);
                final RWIProcess ranking = genSearchresult(prop, sb, segment, keyhash, flags);
                genURLList(prop, keyhash, keystring, ranking, flags, count);
            }
View Full Code Here
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
        for (final String list : lists)
            prop.put("genUrlList_blacklists_" + i++ + "_name", list);
    }

    public static Bitfield compileFlags(final serverObjects post) {
        final Bitfield b = new Bitfield(4);
        if (post.get("allurl", "").equals("on")) return null;
        if (post.get("flags") != null) {
            if (post.get("flags","").length() == 0) return null;
            return new Bitfield(4, post.get("flags"));
        }
        if (post.get("description", "").equals("on")) b.set(WordReferenceRow.flag_app_dc_description, true);
        if (post.get("title", "").equals("on")) b.set(WordReferenceRow.flag_app_dc_title, true);
        if (post.get("creator", "").equals("on")) b.set(WordReferenceRow.flag_app_dc_creator, true);
        if (post.get("subject", "").equals("on")) b.set(WordReferenceRow.flag_app_dc_subject, true);
        if (post.get("url", "").equals("on")) b.set(WordReferenceRow.flag_app_dc_identifier, true);
        if (post.get("emphasized", "").equals("on")) b.set(WordReferenceRow.flag_app_emphasized, true);
        if (post.get("image", "").equals("on")) b.set(Condenser.flag_cat_hasimage, true);
        if (post.get("audio", "").equals("on")) b.set(Condenser.flag_cat_hasaudio, true);
        if (post.get("video", "").equals("on")) b.set(Condenser.flag_cat_hasvideo, true);
        if (post.get("app", "").equals("on")) b.set(Condenser.flag_cat_hasapp, true);
        if (post.get("indexof", "").equals("on")) b.set(Condenser.flag_cat_indexof, true);
        return b;
    }
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
                            } catch (final NumberFormatException e) {
                                Log.logSevere("kelondroRow", "NumberFormatException for celltype_binary, celltype = " + col.celltype + ", encoder = " + col.encoder + ", value = '" + token.substring(p + 1).trim() + "'");
                                setCol(clstrt, col.cellwidth, new byte[]{0});
                            }
                        } else if ((decimalCardinal) && (col.celltype == Column.celltype_bitfield)) {
                            setCol(clstrt, col.cellwidth, (new Bitfield(col.cellwidth, token.substring(p + 1).trim())).bytes());
                        } else {
                            setCol(clstrt, col.cellwidth, UTF8.getBytes(token.substring(p + 1).trim()));
                        }
                    }
                }
View Full Code Here
604
605
606
607
608
609
610
611
612
613
614
                bb.append(propertySymbol);
                if (quotes) bb.append('"');
                if ((decimalCardinal) && (Row.this.row[i].celltype == Column.celltype_cardinal)) {
                    bb.append(Long.toString(getColLong(i)));
                } else if ((decimalCardinal) && (Row.this.row[i].celltype == Column.celltype_bitfield)) {
                    bb.append((new Bitfield(getColBytes(i, true))).exportB64());
                } else if ((decimalCardinal) && (Row.this.row[i].celltype == Column.celltype_binary)) {
                    assert Row.this.row[i].cellwidth == 1 : toString();
                    bb.append(Integer.toString((0xff & getColByte(i))));
                } else {
                    bb.append(this.rowinstance, this.offset + Row.this.colstart[i], Row.this.row[i].cellwidth);
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
        // if addMedia == true, then all the media links are also parsed and added to the words
        // added media words are flagged with the appropriate media flag
        this.intStringFormatter.setMinimumIntegerDigits(numlength);
        this.intStringFormatter.setMaximumIntegerDigits(numlength);
        this.words = new HashMap<String, Word>();
        this.RESULT_FLAGS = new Bitfield(4);

        // construct flag set for document
        if (!document.getImages().isEmpty())     this.RESULT_FLAGS.set(flag_cat_hasimage, true);
        if (!document.getAudiolinks().isEmpty()) this.RESULT_FLAGS.set(flag_cat_hasaudio, true);
        if (!document.getVideolinks().isEmpty()) this.RESULT_FLAGS.set(flag_cat_hasvideo, true);
View Full Code Here
261
262
263
264
265
266
267
    public int urlcomps() {
        return (int) this.entry.getColLong(col_urlComps);
    }
   
    public Bitfield flags() {
        return new Bitfield(this.entry.getColBytes(col_flags, true));
    }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
     * @return a ranking: the higher the number, the better is the ranking
     */
    public long cardinal(final WordReferenceVars t) {
        //return Long.MAX_VALUE - preRanking(ranking, iEntry, this.entryMin, this.entryMax, this.searchWords);
        // the normalizedEntry must be a normalized indexEntry
        final Bitfield flags = t.flags();
        assert this.min != null;
        assert this.max != null;
        assert t != null;
        assert this.ranking != null;
        final long tf = ((this.max.termFrequency() == this.min.termFrequency()) ? 0 : (((int)(((t.termFrequency()-this.min.termFrequency())*256.0)/(this.max.termFrequency() - this.min.termFrequency())))) << this.ranking.coeff_termfrequency);
        //System.out.println("tf(" + t.urlHash + ") = " + Math.floor(1000 * t.termFrequency()) + ", min = " + Math.floor(1000 * min.termFrequency()) + ", max = " + Math.floor(1000 * max.termFrequency()) + ", tf-normed = " + tf);
        final int maxmaxpos = this.max.maxposition();
        final int minminpos = this.min.minposition();
        final long r =
             ((256 - DigestURI.domLengthNormalized(t.urlhash())) << this.ranking.coeff_domlength)
           + ((this.ranking.coeff_ybr > 12) ? ((256 - (BlockRank.ranking(t.urlhash()) << 4)) << this.ranking.coeff_ybr) : 0)
           + ((this.max.urlcomps()      == this.min.urlcomps()   )   ? 0 : (256 - (((t.urlcomps()     - this.min.urlcomps()     ) << 8) / (this.max.urlcomps()     - this.min.urlcomps())     )) << this.ranking.coeff_urlcomps)
           + ((this.max.urllength()     == this.min.urllength()  )   ? 0 : (256 - (((t.urllength()    - this.min.urllength()    ) << 8) / (this.max.urllength()    - this.min.urllength())    )) << this.ranking.coeff_urllength)
           + ((maxmaxpos == minminpos)                               ? 0 : (256 - (((t.minposition() - minminpos) << 8) / (maxmaxpos - minminpos))) << this.ranking.coeff_posintext)
           + ((this.max.posofphrase()   == this.min.posofphrase())   ? 0 : (256 - (((t.posofphrase()  - this.min.posofphrase()  ) << 8) / (this.max.posofphrase()  - this.min.posofphrase())  )) << this.ranking.coeff_posofphrase)
           + ((this.max.posinphrase()   == this.min.posinphrase())   ? 0 : (256 - (((t.posinphrase()  - this.min.posinphrase()  ) << 8) / (this.max.posinphrase()  - this.min.posinphrase())  )) << this.ranking.coeff_posinphrase)
           + ((this.max.distance()      == this.min.distance()   )   ? 0 : (256 - (((t.distance()     - this.min.distance()     ) << 8) / (this.max.distance()     - this.min.distance())     )) << this.ranking.coeff_worddistance)
           + ((this.max.virtualAge()    == this.min.virtualAge())    ? 0 :        (((t.virtualAge()   - this.min.virtualAge()   ) << 8) / (this.max.virtualAge()   - this.min.virtualAge())    ) << this.ranking.coeff_date)
           + ((this.max.wordsintitle()  == this.min.wordsintitle())  ? 0 : (((t.wordsintitle() - this.min.wordsintitle()  ) << 8) / (this.max.wordsintitle() - this.min.wordsintitle())  ) << this.ranking.coeff_wordsintitle)
           + ((this.max.wordsintext()   == this.min.wordsintext())   ? 0 : (((t.wordsintext()  - this.min.wordsintext()   ) << 8) / (this.max.wordsintext()  - this.min.wordsintext())   ) << this.ranking.coeff_wordsintext)
           + ((this.max.phrasesintext() == this.min.phrasesintext()) ? 0 : (((t.phrasesintext()- this.min.phrasesintext() ) << 8) / (this.max.phrasesintext()- this.min.phrasesintext()) ) << this.ranking.coeff_phrasesintext)
           + ((this.max.llocal()        == this.min.llocal())        ? 0 : (((t.llocal()       - this.min.llocal()        ) << 8) / (this.max.llocal()       - this.min.llocal())        ) << this.ranking.coeff_llocal)
           + ((this.max.lother()        == this.min.lother())        ? 0 : (((t.lother()       - this.min.lother()        ) << 8) / (this.max.lother()       - this.min.lother())        ) << this.ranking.coeff_lother)
           + ((this.max.hitcount()      == this.min.hitcount())      ? 0 : (((t.hitcount()     - this.min.hitcount()      ) << 8) / (this.max.hitcount()     - this.min.hitcount())      ) << this.ranking.coeff_hitcount)
           + tf
           + ((this.ranking.coeff_authority > 12) ? (authority(t.hosthash()) << this.ranking.coeff_authority) : 0)
           + ((flags.get(WordReferenceRow.flag_app_dc_identifier))  ? 255 << this.ranking.coeff_appurl             : 0)
           + ((flags.get(WordReferenceRow.flag_app_dc_title))       ? 255 << this.ranking.coeff_app_dc_title       : 0)
           + ((flags.get(WordReferenceRow.flag_app_dc_creator))     ? 255 << this.ranking.coeff_app_dc_creator     : 0)
           + ((flags.get(WordReferenceRow.flag_app_dc_subject))     ? 255 << this.ranking.coeff_app_dc_subject     : 0)
           + ((flags.get(WordReferenceRow.flag_app_dc_description)) ? 255 << this.ranking.coeff_app_dc_description : 0)
           + ((flags.get(WordReferenceRow.flag_app_emphasized))     ? 255 << this.ranking.coeff_appemph            : 0)
           + ((flags.get(Condenser.flag_cat_indexof))      ? 255 << this.ranking.coeff_catindexof         : 0)
           + ((flags.get(Condenser.flag_cat_hasimage))     ? 255 << this.ranking.coeff_cathasimage        : 0)
           + ((flags.get(Condenser.flag_cat_hasaudio))     ? 255 << this.ranking.coeff_cathasaudio        : 0)
           + ((flags.get(Condenser.flag_cat_hasvideo))     ? 255 << this.ranking.coeff_cathasvideo        : 0)
           + ((flags.get(Condenser.flag_cat_hasapp))       ? 255 << this.ranking.coeff_cathasapp          : 0)
           + ((ByteBuffer.equals(t.language, this.language)) ? 255 << this.ranking.coeff_language           : 0)
           + ((DigestURI.probablyRootURL(t.urlhash())) ?  15 << this.ranking.coeff_urllength          : 0);

        //if (searchWords != null) r += (yacyURL.probablyWordURL(t.urlHash(), searchWords) != null) ? 256 << ranking.coeff_appurl : 0;

View Full Code Here
TOP

Related Classes of net.yacy.kelondro.order.Bitfield

  • de.anomic.crawler.ResultURLs
  • de.anomic.crawler.retrieval.Request
  • de.anomic.search.ReferenceOrder
  • IndexControlRWIs_p
  • net.yacy.document.Condenser
  • net.yacy.kelondro.data.image.ImageReferenceRow
  • net.yacy.kelondro.data.meta.URIMetadataRow
  • net.yacy.kelondro.data.word.WordReferenceRow
  • net.yacy.kelondro.index.Row$Entry
  • net.yacy.search.ranking.ReferenceOrder
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.