Package krati.util

Examples of krati.util.Chronos

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
    } catch(Exception e) {
      e.printStackTrace();
      System.exit(1);
    }
   
    Chronos c = new Chronos();
    NetworkTypeaheadConfig<TypeaheadElement> config;
    WeightedNetworkTypeaheadInitializer<TypeaheadElement> initializer;
   
    /**
     * Configure source NetworkTypeahead
     */
    config = TypeaheadConfigFactory.createNetworkTypeaheadConfig(sourceProperties);
    config.setElementStoreCached(false); // disable cache to bypass serialization
    config.setElementStoreSegmentFactory(new MappedSegmentFactory());
    config.setConnectionsStoreSegmentFactory(new MappedSegmentFactory());
    initializer = new WeightedNetworkTypeaheadInitializer<TypeaheadElement>(config);
   
    WeightedNetworkTypeahead<TypeaheadElement> sourceTypeahead =
      (WeightedNetworkTypeahead<TypeaheadElement>) initializer.getTypeahead();
    long sourceLoadTime = c.tick();
   
    /**
     * Configure target NetworkTypeahead
     */
    config = TypeaheadConfigFactory.createNetworkTypeaheadConfig(targetProperties);
    config.setElementStoreCached(false); // disable cache to bypass serialization
    config.setElementStoreSegmentFactory(new MappedSegmentFactory());
    config.setConnectionsStoreSegmentFactory(new WriteBufferSegmentFactory(config.getConnectionsStoreSegmentMB()));
    initializer = new WeightedNetworkTypeaheadInitializer<TypeaheadElement>(config);
   
    WeightedNetworkTypeahead<TypeaheadElement> targetTypeahead =
      (WeightedNetworkTypeahead<TypeaheadElement>) initializer.getTypeahead();
    long targetLoadTime = c.tick();
   
    // Set target NetworkTypehead connections store hwMark before copying.
    targetTypeahead.getConnectionsStore().saveHWMark(targetConnectionsStoreHWMark);
   
    // Copy now!
    Copy.copy(sourceTypeahead, targetTypeahead);
    long copyTime = c.tick();
   
    // Set target NetworkTypehead connections store hwMark after copying and then sync.
    targetTypeahead.getConnectionsStore().saveHWMark(targetConnectionsStoreHWMark);
    targetTypeahead.getConnectionsStore().sync();
   
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
  public ConnectionsScanner(File connsHomeDir) {
    this.connsHomeDir = connsHomeDir;
  }
 
  public void scan(ConnectionsHandler handler) {
    Chronos c = new Chronos();
   
    for(File f : connsHomeDir.listFiles()) {
      if(f.isFile()) {
        try {
          scan(f, handler);
          System.out.printf("%s scanned in %d ms%n", f.getAbsolutePath(), c.tick());
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    this.scoreDir = scoreDir;
  }
 
  @Override
  public void scan(ScoreHandler handler) {
    Chronos c = new Chronos();
    if(scoreDir == null || !scoreDir.exists()) return;
     
    if(scoreDir.isDirectory()) {
      for(File f : scoreDir.listFiles()) {
        if(f.isFile()) {
          try {
            scan(f, handler);
            System.out.printf("%s scanned in %d ms%n", f.getAbsolutePath(), c.tick());
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      }
    } else if(scoreDir.isFile()) {
      try {
        scan(scoreDir, handler);
        System.out.printf("%s scanned in %d ms%n", scoreDir.getAbsolutePath(), c.tick());
      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
  public TermsScanner(File termsDir) {
    this.termsDir = termsDir;
  }
 
  public void scan(TermsHandler handler) {
    Chronos c = new Chronos();
   
    if(termsDir.exists()) {
      if(termsDir.isDirectory()) {
        for(File f : termsDir.listFiles()) {
          if(f.isFile()) {
            try {
              scan(f, handler);
              System.out.printf("%s scanned in %d ms%n", f.getAbsolutePath(), c.tick());
            } catch (Exception e) {
              e.printStackTrace();
            }
          }
        }
      } else if(termsDir.isFile()) {
        File f = termsDir;
        try {
          scan(f, handler);
          System.out.printf("%s scanned in %d ms%n", f.getAbsolutePath(), c.tick());
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
    } else {
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
     * @param file
     * @throws IOException
     */
    public final void save(File file) throws IOException {
        _entryFile = file;
        Chronos c = new Chronos();
        DataWriter out = new FastDataWriter(file);

        try {
            out.open();

            // Save entry head
            out.writeLong(STORAGE_VERSION); // write the entry file version
            out.writeLong(_minScn);
            out.writeLong(_maxScn);
            out.writeInt(size());

            // Save entry body
            saveDataSection(out);

            // Save entry tail.
            out.writeLong(_minScn);
            out.writeLong(_maxScn);
        } finally {
            out.close();
        }
       
        if(_log.isInfoEnabled()) {
            _log.info("Saved entry: minScn=" + _minScn + " maxScn=" + _maxScn + " size=" + size() + " file=" + file.getName() + " in " + c.getElapsedTime());
        }
    }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
     * @param file
     * @throws IOException
     */
    public void load(File file) throws IOException {
        _entryFile = file;
        Chronos c = new Chronos();
        ChannelReader in = new ChannelReader(file);

        try {
            in.open();

            // Read entry head
            long fileVersion = in.readLong();
            if (fileVersion != STORAGE_VERSION) {
                throw new RuntimeException("Wrong storage version " + fileVersion + " encounted in " + file.getAbsolutePath() + ". Version " + STORAGE_VERSION
                        + " expected.");
            }

            long minScnHead = in.readLong();
            long maxScnHead = in.readLong();
            int length = in.readInt();

            // Read entry body
            loadDataSection(in, length);

            // Read entry tail
            long minScnTail = in.readLong();
            long maxScnTail = in.readLong();

            if (minScnHead != minScnTail) {
                throw new IOException("min scns don't match: " + minScnHead + " vs " + minScnTail);
            }
            if (maxScnHead != maxScnTail) {
                throw new IOException("max scns don't match:" + maxScnHead + " vs " + maxScnTail);
            }

            _minScn = minScnHead;
            _maxScn = maxScnHead;

            if(_log.isInfoEnabled()) {
                _log.info("loaded entry: minScn=" + _minScn + " maxScn=" + _maxScn + " size=" + size() + " file=" + file.getName() + " in " + c.getElapsedTime());
            }
        } finally {
            in.close();
        }
    }
View Full Code Here
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
       
        Segment segSource = segment;
        int segSourceId = segSource.getSegmentId();
        int segTargetId = segTarget.getSegmentId();
       
        Chronos c = new Chronos();
        if(!segment.canReadFromBuffer() && segment.getLoadFactor() > 0.1) {
            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
       
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = sibEnabled ? _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId) : null;
       
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
       
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
           
            for(int index = 0, cnt = _dataArray.length(); index < cnt; index++) {
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
               
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                   
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                   
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                   
                    if(sibTarget != null) sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
           
            // Push whatever left into update queue
            _updateManager.endUpdate(segTarget);
            _log.info("bytes transferred from " + segSource.getSegmentId() + ": " + bytesTransferred + " time: " + c.tick() + " ms");
           
            return succ;
        } finally {
            if(segSource.getClass() == BufferedSegment.class) {
                segSource.close(false);
View Full Code Here
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
    private boolean compact(Segment segment, SegmentIndexBuffer sibSource, Segment segTarget) throws IOException {
        Segment segSource = segment;
        int segSourceId = segSource.getSegmentId();
        int segTargetId = segTarget.getSegmentId();
       
        Chronos c = new Chronos();
        if(!segment.canReadFromBuffer() && segment.getLoadFactor() > 0.1) {
            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
       
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId);
       
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
       
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
            SegmentIndexBuffer.IndexOffset reuse = new SegmentIndexBuffer.IndexOffset();
           
            for(int i = 0, cnt = sibSource.size(); i < cnt; i++) {
                sibSource.get(i, reuse);
                int index = reuse.getIndex();
                int sibSegPos = reuse.getOffset();
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
               
                if(sibSegPos != oldSegPos) continue;
               
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
               
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                   
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                   
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                   
                    sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
           
            // Push whatever left into update queue
            _updateManager.endUpdate(segTarget);
            _log.info("bytes fastscanned from " + segSource.getSegmentId() + ": " + bytesTransferred + " time: " + c.tick() + " ms");
           
            return succ;
        } finally {
            if(segSource.getClass() == BufferedSegment.class) {
                segSource.close(false);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
                url, source, keySerializer, valueSerializer);
       
        String ret;
        String key;
        GenericRecord value;
        Chronos c = new Chronos();
       
        // initialize store
        ret = client.send(StoreDirective.StoreInit);
        System.out.println(StoreDirective.StoreInit + ": " + ret);
        System.out.println(c.getElapsedTime());
       
        // schema
        System.out.println(schema.toString());
       
        // put
        for(int i = 0; i < 100; i++) {
            key = "member." + i;
            value = createRecord(schema, i);
            client.put(key, value);
            System.out.println("put: " + key + "->"  + value);
        }
        System.out.println(c.getElapsedTime());
       
        // Sync changes to store
        client.send(StoreDirective.StoreSync);
        System.out.println(StoreDirective.StoreSync + ": " + ret);
        System.out.println(c.getElapsedTime());
       
        // get
        for(int i = 0; i < 100; i++) {
            key = "member." + i;
            value = client.get(key);
            System.out.println("get: " + key + "->"  + value);
        }
        System.out.println(c.getElapsedTime());
       
        // multi-put
        Map<String, GenericRecord> mputMap = new HashMap<String, GenericRecord>();
        for(int i = 100; i < 200; i++) {
            key = "member." + i;
            value = createRecord(schema, i);
            mputMap.put(key, value);
        }
       
        client.put(mputMap);
        System.out.println("mput: " + c.getElapsedTime() + " " + mputMap.size() + " records");
       
        // multi-get
        Map<String, GenericRecord> mgetMap = client.get(mputMap.keySet());
        System.out.println("mget: " + c.getElapsedTime() + " " + mgetMap.size() + " records");
    }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
                url, source, keySerializer, valueSerializer);
       
        String ret;
        String key;
        GenericRecord value;
        Chronos c = new Chronos();
       
        // initialize store
        ret = client.send(StoreDirective.StoreInit);
        System.out.println(StoreDirective.StoreInit + ": " + ret);
        System.out.println(c.getElapsedTime());
       
        // schema
        System.out.println(schema.toString());
       
        // put
        for(int i = 0; i < 100; i++) {
            key = "member." + i;
            value = createRecord(schema, i);
            client.put(key, value);
            System.out.println("put: " + key + "->"  + value);
        }
        System.out.println(c.getElapsedTime());
       
        // Sync changes to store
        client.send(StoreDirective.StoreSync);
        System.out.println(StoreDirective.StoreSync + ": " + ret);
        System.out.println(c.getElapsedTime());
       
        // get
        for(int i = 0; i < 100; i++) {
            key = "member." + i;
            value = client.get(key);
            System.out.println("get: " + key + "->"  + value);
        }
        System.out.println(c.getElapsedTime());
       
        // multi-put
        Map<String, GenericRecord> mputMap = new HashMap<String, GenericRecord>();
        for(int i = 100; i < 200; i++) {
            key = "member." + i;
            value = createRecord(schema, i);
            mputMap.put(key, value);
        }
       
        client.put(mputMap);
        System.out.println("mput: " + c.getElapsedTime() + " " + mputMap.size() + " records");
       
        // multi-get
        Map<String, GenericRecord> mgetMap = client.get(mputMap.keySet());
        System.out.println("mget: " + c.getElapsedTime() + " " + mgetMap.size() + " records");
    }
View Full Code Here
TOP

Related Classes of krati.util.Chronos

  • cleo.search.bootstrap.ArrayStoreConnectionsBootstrap
  • cleo.search.bootstrap.GenericTypeaheadBootstrap
  • cleo.search.bootstrap.GenericTypeaheadConnectionsStoreBootstrap
  • cleo.search.bootstrap.NetworkTypeaheadConnectionsStoreBootstrap
  • cleo.search.bootstrap.NetworkTypeaheadWeightedConnectionsStoreBootstrap
  • cleo.search.bootstrap.NetworkTypeaheadWeightedConnectionsStoreMaxScn
  • cleo.search.bootstrap.TypeaheadElementStoreBootstrap
  • cleo.search.copy.CopyVanillaNetworkTypeahead
  • cleo.search.copy.CopyWeightedNetworkTypeahead
  • cleo.search.typeahead.GenericTypeaheadIndexRoller
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.