Package org.syncany.plugins.unreliable_local

Examples of org.syncany.plugins.unreliable_local.UnreliableLocalTransferSettings


  }

  @Test
  public void testUnreliableUpload_Test1_WithRetryFailsManyTimes() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            // 1st upload (= multichunk) fails
            "rel=[345] .+upload.+multichunk", // << 3 retries!!

            // Make fourth upload fail
            "rel=(8|9|10) .+upload" // << 3 retries!!
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    int i = 0;
    while (i++ < 5) {
      clientA.createNewFile("A-original-" + i, 50 * 1024);

      try {
        Thread.sleep(100);
        clientA.up();
        clientA.down();
      }
      catch (StorageException e) {
        logger.log(Level.INFO, e.getMessage());
      }
    }

    assertTrue(new File(testConnection.getPath() + "/databases/database-A-0000000001").exists());
    assertTrue(new File(testConnection.getPath() + "/databases/database-A-0000000002").exists());
    assertTrue(new File(testConnection.getPath() + "/databases/database-A-0000000003").exists());
    assertFalse(new File(testConnection.getPath() + "/databases/database-A-0000000004").exists());
    assertFalse(new File(testConnection.getPath() + "/databases/database-A-0000000005").exists());

    // Tear down
    clientA.deleteTestData();
  }
View Full Code Here


    /*
     * This test fails when trying to upload the first multichunk.
     */

    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            ".+upload.+multichunk",
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("A-original", 10);

    boolean upFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      upFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(upFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);

    assertEquals(2, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("database-"));
View Full Code Here

     * 6. move(temp-1, multichunks/multichunk-1)
     * 8. move(temp-2, databases/database-123)
     */

    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            "rel=[456].+upload.+database",
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    // 1. First upload fails
    clientA.createNewFile("A-original", 10);

    boolean upFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      upFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(upFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    assertEquals(1, new File(testConnection.getPath() + "/temporary/").listFiles().length);

    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);

    assertEquals(2, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("database-"));

    // 2. Second try succeeds and must clean up the transactions
    clientA.up();

    assertEquals(1, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    assertEquals(0, new File(testConnection.getPath() + "/temporary/").listFiles().length);

    // Tear down
    clientA.deleteTestData();
  }
View Full Code Here

     * 7. move(temp-2, multichunks/multichunk-2)
     * 8. move(temp-3, databases/database-123)
     */

    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            "rel=[456].+upload.+multichunk",
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("A-original", 5 * 1024 * 1024); // << larger than one multichunk!

    boolean upFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      upFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(upFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    File[] tempFiles = new File(testConnection.getPath() + "/temporary/").listFiles();

    assertEquals(1, tempFiles.length);
    assertTrue(tempFiles[0].length() > 500 * 1024); // 1 MC with 1 MB, 1 with 4 MB; must be larger than 500 KB

    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);

    assertEquals(3, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(2).getRemoteFile().getName().contains("database-"));

    // 2. Second try succeeds and must clean up the transactions
    clientA.up();

    assertEquals(1, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(2, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    assertEquals(0, new File(testConnection.getPath() + "/temporary/").listFiles().length);

    // Tear down
    clientA.deleteTestData();
  }
View Full Code Here

     * 7. move(temp-2, multichunks/multichunk-2) <<< FAILS HERE (first run)
     * 8. move(temp-3, databases/database-123)
     */

    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            "rel=[234].+move.+multichunk",
            "rel=(8|9|10).+upload.+transaction",
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("A-original", 5 * 1024 * 1024); // << larger than one multichunk!

    boolean firstUpFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      firstUpFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(firstUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    File[] tempFiles = new File(testConnection.getPath() + "/temporary/").listFiles();

    assertEquals(2, tempFiles.length);
    assertTrue(tempFiles[0].length() > 500 * 1024 || tempFiles[1].length() > 500 * 1024); // The second multichunk should be >500 KB
    assertTrue(tempFiles[0].length() < 100 * 1024 || tempFiles[1].length() < 100 * 1024); // The database file should be <100 KB

    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);

    assertEquals(3, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(2).getRemoteFile().getName().contains("database-"));

    // 2. Double check if list() does not return the multichunk
    TransferManager transferManager = new TransactionAwareTransferManager(
        new UnreliableLocalTransferPlugin().createTransferManager(testConnection, null), null);
    Map<String, MultichunkRemoteFile> multiChunkList = transferManager.list(MultichunkRemoteFile.class);
    assertEquals(0, multiChunkList.size());

    // 3. Second try fails in the beginning, to see if cleanTransactions was successful
    boolean secondUpFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      secondUpFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(secondUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(2, new File(testConnection.getPath() + "/actions/").listFiles().length); // Shouldn't this be 1
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    assertEquals(0, new File(testConnection.getPath() + "/").listFiles(new FilenameFilter() {
      public boolean accept(File dir, String name) {
        return name.contains("temp-");
      }
    }).length);

View Full Code Here

     *
     * Expected: temp-(83,85,86)
     */

    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>

            "rel=[234].+move.+multichunk",
            "rel=[234].+delete.+temp",
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("A-original", 5 * 1024 * 1024); // << larger than one multichunk!

    boolean firstUpFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      firstUpFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(firstUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);

    File[] tempFiles = new File(testConnection.getPath() + "/temporary").listFiles();

    assertEquals(2, tempFiles.length);
    assertTrue(tempFiles[0].length() > 500 * 1024 || tempFiles[1].length() > 500 * 1024); // The second multichunk should be >500 KB
    assertTrue(tempFiles[0].length() < 100 * 1024 || tempFiles[1].length() < 100 * 1024); // The database file should be <100 KB

    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);

    assertEquals(3, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(2).getRemoteFile().getName().contains("database-"));

    // 2. Double check if list() does not return the multichunk
    TransferManager transferManager = new TransactionAwareTransferManager(
        new UnreliableLocalTransferPlugin().createTransferManager(testConnection, null), null);
    Map<String, MultichunkRemoteFile> multiChunkList = transferManager.list(MultichunkRemoteFile.class);
    assertEquals(0, multiChunkList.size());

    // 3. Second try fails in the beginning, to see if cleanTransactions was successful
    boolean secondUpFailed = false;

    try {
      clientA.up();
    }
    catch (StorageException e) {
      secondUpFailed = true;
      logger.log(Level.INFO, e.getMessage());
    }

    assertTrue(secondUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(2, new File(testConnection.getPath() + "/actions/").listFiles().length); // left over, 2 failed ops
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    assertEquals(4, new File(testConnection.getPath() + "/temporary/").listFiles().length);

    // 4. Third try; this should finally succeed
    clientA.up();

    assertEquals(1, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(2, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/actions/").listFiles().length); // cleaned
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    assertEquals(4, new File(testConnection.getPath() + "/temporary/").listFiles().length); // cleaned

    // Tear down
    clientA.deleteTestData();
  }
View Full Code Here

  }

  @Test
  public void testUnreliableCleanup_Test1_oldVersionRemoval() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>
            "rel=(11|12|13).+upload.+database", // << 3 retries!!
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("file");

    clientA.up();

    clientA.changeFile("file");
    clientA.upWithForceChecksum();

    CleanupOperationOptions cleanupOptions = new CleanupOperationOptions();
    cleanupOptions.setKeepVersionsCount(1);
    boolean cleanupFailed = false;
    try {
      clientA.cleanup(cleanupOptions);
    }
    catch (StorageException e) {
      cleanupFailed = true;
    }

    assertTrue(cleanupFailed);
    TransferManager transferManager = new TransactionAwareTransferManager(
        new UnreliableLocalTransferPlugin().createTransferManager(testConnection, null), null);
    assertEquals(2, transferManager.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);
    assertEquals(2, transferManager.list(DatabaseRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "databases").list().length);
    assertEquals(1, transferManager.list(TransactionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "transactions").list().length);
    assertEquals(1, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "actions").list().length);

    clientA.cleanup(cleanupOptions);
    assertEquals(1, transferManager.list(MultichunkRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "multichunks").list().length);
    assertEquals(3, transferManager.list(DatabaseRemoteFile.class).size());
    assertEquals(3, new File(testConnection.getPath(), "databases").list().length);
    assertEquals(0, transferManager.list(TransactionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "transactions").list().length);
    assertEquals(0, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);

    clientA.deleteTestData();
  }
View Full Code Here

  }

  @Test
  public void testUnreliableCleanup_Test2_databaseFileMerge() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>
            "rel=(11|12|13).+upload.+database", // << 3 retries!!
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("file");

    clientA.up();

    clientA.changeFile("file");
    clientA.upWithForceChecksum();

    CleanupOperationOptions cleanupOptions = new CleanupOperationOptions();
    cleanupOptions.setMaxDatabaseFiles(1);
    boolean cleanupFailed = false;
    try {
      clientA.cleanup(cleanupOptions);
    }
    catch (StorageException e) {
      cleanupFailed = true;
    }

    assertTrue(cleanupFailed);
    TransferManager transferManager = new TransactionAwareTransferManager(
        new UnreliableLocalTransferPlugin().createTransferManager(testConnection, null), null);
    assertEquals(2, transferManager.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);

    // Note that the list here differs from the actual files, because the transaction fails
    // while deletions have been done
    assertEquals(2, transferManager.list(DatabaseRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "databases").list().length);

    assertEquals(1, transferManager.list(TransactionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "transactions").list().length);

    assertEquals(1, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "actions").list().length);

    File tempDir = TestFileUtil.createTempDirectoryInSystemTemp();
    File tempFile = File.createTempFile("multichunk", "", tempDir);
    for (RemoteFile remoteFile : transferManager.list(DatabaseRemoteFile.class).values()) {
      transferManager.download(remoteFile, tempFile);
      assertTrue(tempFile.exists());
      tempFile.delete();
    }

    clientA.cleanup(cleanupOptions);

    assertEquals(2, transferManager.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);

    assertEquals(1, transferManager.list(DatabaseRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "databases").list().length);

    assertEquals(0, transferManager.list(TransactionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "transactions").list().length);

    assertEquals(0, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);
  }
View Full Code Here

  }

  @Test
  public void testUnreliableCleanup_Test3_unreferencedTempFiles() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(
        Arrays.asList(new String[] {
            // List of failing operations (regex)
            // Format: abs=<count> rel=<count> op=<connect|init|upload|...> <operation description>
            "rel=[678].+delete.+temp", // << 3 retries!!
        }
            ));

    TestClient clientA = new TestClient("A", testConnection);

    clientA.createNewFile("file");

    clientA.up();

    clientA.changeFile("file");
    clientA.upWithForceChecksum();

    CleanupOperationOptions cleanupOptions = new CleanupOperationOptions();
    cleanupOptions.setMaxDatabaseFiles(1);
    boolean cleanupFailed = false;
    try {
      clientA.cleanup(cleanupOptions);
    }
    catch (StorageException e) {
      cleanupFailed = true;
    }

    TransferManager transferManager = new TransactionAwareTransferManager(
        new UnreliableLocalTransferPlugin().createTransferManager(testConnection, null), null);

    assertTrue(cleanupFailed);
    assertEquals(2, transferManager.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);

    assertEquals(1, transferManager.list(DatabaseRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "databases").list().length);

    assertEquals(0, transferManager.list(TransactionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "transactions").list().length);

    assertEquals(1, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "actions").list().length);

    // Two temporary files left (first deletion failed)
    assertEquals(2, transferManager.list(TempRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "temporary").list().length);

    clientA.cleanup(cleanupOptions);

    // Functional cleanup results in removal of action file and unreferenced files
    assertEquals(0, transferManager.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);

    assertEquals(0, transferManager.list(TempRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "temporary").list().length);
  }
View Full Code Here

  public static InitOperationOptions createTestUnreliableInitOperationOptions(String machineName, List<String> failingOperationPatterns)
      throws Exception {
    InitOperationOptions initOperationOptions = createTestInitOperationOptions(machineName);
    // createTestInitOperationOptions always returns LocalTransferSettings
    File tempRpoDir = ((LocalTransferSettings) initOperationOptions.getConfigTO().getTransferSettings()).getPath();
    UnreliableLocalTransferSettings transferSettings = Plugins.get("unreliable_local", TransferPlugin.class).createEmptySettings();
    transferSettings.setPath(tempRpoDir);
    transferSettings.setFailingOperationPatterns(failingOperationPatterns);

    initOperationOptions.getConfigTO().setTransferSettings(transferSettings);

    return initOperationOptions;
  }
View Full Code Here

TOP

Related Classes of org.syncany.plugins.unreliable_local.UnreliableLocalTransferSettings

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.