Package org.openstreetmap.osmosis.replication.common

Examples of org.openstreetmap.osmosis.replication.common.ReplicationState

30
31
32
33
34
35
36
37
38
39
40
    server = new ReplicationSequenceServer(0);
    server.setChangeSink(new MockReplicationDestination());

    try {
      for (int i = 0; i < 10; i++) {
        ReplicationState state = new ReplicationState();
        Map<String, Object> metaData = new HashMap<String, Object>();
        metaData.put(ReplicationState.META_DATA_KEY, state);
        server.initialize(metaData);
        Thread.sleep(10);
        server.complete();
View Full Code Here
92
93
94
95
96
97
98
99
100
101
  }


  private ReplicationState getReplicationState(long sequenceNumber) {
    PropertiesPersister persister = new PropertiesPersister(getStateFile(sequenceNumber));
    ReplicationState state = new ReplicationState();
    state.load(persister.loadMap());

    return state;
  }
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
 
 
  private ReplicationState download(ReplicationDownloaderConfiguration configuration, ReplicationState serverState,
      ReplicationState initialLocalState) {
    URL baseUrl;
    ReplicationState localState;
    Date maximumDownloadTimestamp;
   
    localState = initialLocalState;
   
    // Determine the location of download files.
    baseUrl = configuration.getBaseUrl();
   
    // Determine the maximum timestamp that can be downloaded.
    maximumDownloadTimestamp =
      calculateMaximumTimestamp(configuration, serverState.getTimestamp(), localState.getTimestamp());
    LOG.fine("The maximum timestamp to be downloaded is " + maximumDownloadTimestamp + ".");
   
    // Download all files and send their contents to the sink.
    while (localState.getSequenceNumber() < serverState.getSequenceNumber()) {
      File replicationFile;
      long sequenceNumber;
      ReplicationState fileReplicationState;
     
      // Check to see if our local state has already reached the maximum
      // allowable timestamp. This will typically occur if a job is run
      // again before new data becomes available, or if an implementation
      // of this class (eg. ReplicationFileMerger) is waiting for a full
      // time period of data to become available before processing.
      if (localState.getTimestamp().compareTo(maximumDownloadTimestamp) >= 0) {
        break;
      }
     
      // Calculate the next sequence number.
      sequenceNumber = localState.getSequenceNumber() + 1;
      LOG.finer("Processing replication sequence " + sequenceNumber + ".");
     
      // Get the state associated with the next file.
      fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber);
     
      // Ensure that the next state is within the allowable timestamp
      // range. We must stop if the next data takes us beyond the maximum
      // timestamp. This will either occur if a maximum download time
      // duration limit has been imposed, or if a time-aligned boundary
      // has been reached.
      if (fileReplicationState.getTimestamp().compareTo(maximumDownloadTimestamp) > 0) {
        // We will always allow at least one replication interval
        // through to deal with the case where a single interval exceeds
        // the maximum duration. This can happen if the source data has
        // a long time gap between two intervals due to system downtime.
        if (localState.getSequenceNumber() != initialLocalState.getSequenceNumber()) {
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
 
  private void runImpl() {
    try {
      ReplicationDownloaderConfiguration configuration;
      ReplicationState serverState;
      ReplicationState localState;
      PropertiesPersister localStatePersistor;
     
      // Instantiate utility objects.
      configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
     
      // Obtain the server state.
      LOG.fine("Reading current server state.");
      serverState = serverStateReader.getServerState(configuration.getBaseUrl());
     
      // Build the local state persister which is used for both loading and storing local state.
      localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
     
      // Begin processing.
      processInitialize(Collections.<String, Object>emptyMap());
     
      // If local state isn't available we need to copy server state to be the initial local state
      // then exit.
      if (localStatePersistor.exists()) {
        localState = new ReplicationState(localStatePersistor.loadMap());
       
        // Download and process the replication files.
        localState = download(configuration, serverState, localState);
       
      } else {
        localState = serverState;
       
        processInitializeState(localState);
      }
     
      // Commit downstream changes.
      processComplete();
     
      // Persist the local state.
      localStatePersistor.store(localState.store());
     
    } finally {
      processRelease();
    }
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    changeSink.complete();
  }


  private void invokeSinkInit() {
    replicationState = new ReplicationState();
    Map<String, Object> metaData = new HashMap<String, Object>(1);
    metaData.put(ReplicationState.META_DATA_KEY, replicationState);
    changeSink.initialize(metaData);
    sinkInitInvoked = true;
  }
View Full Code Here
111
112
113
114
115
116
117
118
119
120
  }


  private ReplicationState loadState(File stateFile) {
    PropertiesPersister persister = new PropertiesPersister(stateFile);
    ReplicationState state = new ReplicationState();
    state.load(persister.loadMap());

    return state;
  }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
            invokeSinkInit();
          }

          // The first chunk contains the replication state stored in
          // properties format.
          ReplicationState serverReplicationState = loadState(chunkFile);
          if (LOG.isLoggable(Level.FINER)) {
            LOG.finer("Received replication state " + serverReplicationState.getSequenceNumber());
          }

          // Validate that the server has sent us the expected state.
          if (serverReplicationState.getSequenceNumber() != replicationState.getSequenceNumber()) {
            throw new OsmosisRuntimeException("Received sequence number "
                + serverReplicationState.getSequenceNumber() + " from server, expected "
                + replicationState.getSequenceNumber());
          }

          // Update the local state with server values.
          replicationState.setTimestamp(serverReplicationState.getTimestamp());
          replicationStateReceived = true;
         
          // If this is replication 0, then we need to finish
          // processing now because the first sequence doesn't have
          // any data.
View Full Code Here
207
208
209
210
211
212
213
214
215
216
    if (alignedDate.compareTo(initialDate) < 0) {
      alignedDate = new Date(alignedDate.getTime() + intervalLength);
    }

    // Create an initial replication state object.
    currentDataState = new ReplicationState(alignedDate, 0);

    // Write out the initial "0" state file.
    replicationStore.saveState(currentDataState);
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  /**
   * Calculate the replication lag and print it to stdout
   */
  private void getLag() {
    ReplicationDownloaderConfiguration configuration;
    ReplicationState serverState;
    ReplicationState localState;
    PropertiesPersister localStatePersistor;
   
    // Instantiate utility objects.
    configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
   
    // Obtain the server state.
    LOG.fine("Reading current server state.");
    serverState = serverStateReader.getServerState(configuration.getBaseUrl());
   
    // Build the local state persister which is used for both loading and storing local state.
    localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
   
    // If local state isn't available we need to fail because no lag can be calculated.
    if (!localStatePersistor.exists()) {
      throw new OsmosisRuntimeException("Can't read local state.");
    }
   
    // fetch the local state from the file
    localState = new ReplicationState(localStatePersistor.loadMap());
   
    // extract the time of the local and the remote state files
    long local = localState.getTimestamp().getTime();
    long server = serverState.getTimestamp().getTime();
   
    // we assume the server being before the local state while calculating the difference
    long lag = (server - local) / 1000;
   
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
  /**
   * Sends a replication sequence containing dummy data to the destination.
   */
  public void sendSequence() {
    // Initialise the replication stream.
    ReplicationState state = new ReplicationState();
    Map<String, Object> metaData = new HashMap<String, Object>(1);
    metaData.put(ReplicationState.META_DATA_KEY, state);
    changeSink.initialize(metaData);

    // Send the change data unless this is sequence 0 where no data is
    // allowed. We'll only send a single record for simplicity.
    if (state.getSequenceNumber() > 0) {
      // We'll do a create action on the first replication pass, and modify subsequently.
      ChangeAction action;
      if (state.getSequenceNumber() == 1) {
        action = ChangeAction.Create;
      } else {
        action = ChangeAction.Modify;
      }
     
      // Create a change record which data derived from the
      // replication sequence number itself.
      ChangeContainer change = new ChangeContainer(new NodeContainer(new Node(new CommonEntityData(10,
          (int) state.getSequenceNumber(), new Date(state.getSequenceNumber() * 1000), new OsmUser(11,
              "test"), state.getSequenceNumber() * 2), state.getSequenceNumber() * 3,
          state.getSequenceNumber() * 4)), action);
     
      // Send the record downstream.
      changeSink.process(change);
    }
   
    state.setTimestamp(new Date(state.getSequenceNumber() * 1000));
   
    changeSink.complete();
  }
 
View Full Code Here
TOP

Related Classes of org.openstreetmap.osmosis.replication.common.ReplicationState

  • org.openstreetmap.osmosis.core.time.DateFormatter
  • org.openstreetmap.osmosis.core.time.DateParser
  • org.openstreetmap.osmosis.replication.v0_6.BaseReplicationDownloader
  • org.openstreetmap.osmosis.replication.v0_6.ReplicationFileMerger
  • org.openstreetmap.osmosis.replication.v0_6.ReplicationLagReader
  • org.openstreetmap.osmosis.replicationhttp.v0_6.impl.ReplicationDataClientHandler
  • org.openstreetmap.osmosis.replicationhttp.v0_6.impl.ReplicationDataServerHandler
  • org.openstreetmap.osmosis.replicationhttp.v0_6.MockReplicationSource
  • org.openstreetmap.osmosis.replicationhttp.v0_6.ReplicationSequenceServerTest
  • java.util.Date
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.