Examples of org.syncany.chunk.MultiChunk

org.syncany.chunk.MultiChunk
A multichunk represents the container format that stores one to many {@link Chunk}s. Multichunks are created during the chunking/deduplication process by a {@link MultiChunker}.
There are two modes to handle multichunks:
- When a new multichunk is written and filled up with chunks, the {@link Deduper} makes sure thatchunks are only added until a multichunk's minimum size has been reached, and closes the multichunk afterwards. During that process, the {@link #write(Chunk) write()} method is calledfor each chunk, and {@link #isFull()} is checked for the size.
- When a multichunk is read from a file or an input stream, it can be processed sequentially using the {@link #read()} method (not used in current code!), or in a random order using the {@link #getChunkInputStream(byte[]) getChunkInputStream()} method. Because of the latter method,it is essential that random read access on a multichunk is possible.
@author Philipp C. Heckel

  private Set<MultiChunk> chunkFileIntoMultiChunks(File tempDir, List<File> files, Chunker foc, MultiChunker customMultiChunker, Transformer transformer)
      throws IOException {


    Set<MultiChunk> resultMultiChunks = new HashSet<MultiChunk>();


    MultiChunk customMultiChunk = createNewMultiChunk(tempDir, customMultiChunker, transformer);


    for (File file : files) {
      Enumeration<Chunk> chunks = foc.createChunks(file);
      while (chunks.hasMoreElements()) {
        Chunk chunk = chunks.nextElement();
        customMultiChunk.write(chunk);


        if (customMultiChunk.isFull()) {
          customMultiChunk.close();
          resultMultiChunks.add(customMultiChunk);
          customMultiChunk = createNewMultiChunk(tempDir, customMultiChunker, transformer);
        }
      }
    }
    customMultiChunk.close();
    resultMultiChunks.add(customMultiChunk);


    return resultMultiChunks;
  }

View Full Code Here


  private MultiChunk createNewMultiChunk(File tempDir, MultiChunker customMultiChunker, Transformer transformer) {
    FileOutputStream fos;
    MultiChunkId multiChunkId = MultiChunkId.secureRandomMultiChunkId();


    MultiChunk customChunk = null;
    try {
      fos = new FileOutputStream(tempDir.getAbsolutePath() + "/MultiChunk" + multiChunkId);
      customChunk = customMultiChunker.createMultiChunk(multiChunkId, transformer.createOutputStream(fos));
    }
    catch (FileNotFoundException e) {
      e.printStackTrace();
    }
    catch (IOException e) {
      e.printStackTrace();
    }
    
    assertEquals(customChunk.getId(), multiChunkId);


    return customChunk;
  }

View Full Code Here

    Map<ChunkChecksum, File> extractedChunks = new HashMap<ChunkChecksum, File>();
    
    for (File outputMultiChunkFile : outputMultiChunkFiles) {
      logger.log(Level.INFO, "- Extracting multichunk "+outputMultiChunkFile+" ...");
      
      MultiChunk outputMultiChunk = combination.multiChunker.createMultiChunk(
          combination.transformer.createInputStream(new FileInputStream(outputMultiChunkFile)));
      
      Chunk outputChunkInMultiChunk = null;
      
      while (null != (outputChunkInMultiChunk = outputMultiChunk.read())) {
        File extractedChunkFile = new File(tempDir+"/chunk-"+StringUtil.toHex((outputChunkInMultiChunk.getChecksum()))+"-from-multichunk-"+outputMultiChunk.getId());


        logger.log(Level.INFO, "  + Writing chunk "+StringUtil.toHex((outputChunkInMultiChunk.getChecksum()))+" to "+extractedChunkFile+" ...");
        TestFileUtil.writeToFile(outputChunkInMultiChunk.getContent(), extractedChunkFile);


        extractedChunks.put(new ChunkChecksum(outputChunkInMultiChunk.getChecksum()), extractedChunkFile);

View Full Code Here

          multiChunkIdForChunk = memoryDatabase.getMultiChunkIdForChunk(chunkChecksum);
        }


        File decryptedMultiChunkFile = config.getCache().getDecryptedMultiChunkFile(multiChunkIdForChunk);


        MultiChunk multiChunk = multiChunker.createMultiChunk(decryptedMultiChunkFile);
        InputStream chunkInputStream = multiChunk.getChunkInputStream(chunkChecksum.getBytes());


            byte[] buffer = new byte[4096];
            int read = 0;


            while (-1 != (read = chunkInputStream.read(buffer))) {
              reconstructedFileChecksum.update(buffer, 0, read);
                reconstructedFileOutputStream.write(buffer, 0, read);
            }
            
        chunkInputStream.close();
        multiChunk.close();
      }
    }


    reconstructedFileOutputStream.close();

View Full Code Here

    
    // Read chunk from original multichunk
    File repoMultiChunksFolder = new File(testConnection.getPath(), "multichunks");
    File multiChunkFile = repoMultiChunksFolder.listFiles()[0];
    
    MultiChunk multiChunk = clientA.getConfig().getMultiChunker().createMultiChunk(
      clientA.getConfig().getTransformer().createInputStream(new FileInputStream(multiChunkFile)));    
    Chunk chunk = multiChunk.read();
    multiChunk.close();
    
    // Flip byte in chunk and write new "altered" multichunk
    File alteredMultiChunkFile = new File(multiChunkFile + "-altered");
    MultiChunk alteredMultiChunk = clientA.getConfig().getMultiChunker().createMultiChunk(
      multiChunk.getId(), clientA.getConfig().getTransformer().createOutputStream(new FileOutputStream(alteredMultiChunkFile)));
    
    chunk.getContent()[0] ^= 0x01; // Flip one byte!
    alteredMultiChunk.write(chunk);
    alteredMultiChunk.close();
    
    // Now delete old multichunk, and swap by "altered" file
    multiChunkFile.delete();
    FileUtils.moveFile(alteredMultiChunkFile, multiChunkFile);

View Full Code Here

TOP

Related Classes of org.syncany.chunk.MultiChunk

org.syncany.operations.Assembler

org.syncany.tests.chunk.FrameworkCombinationTest

org.syncany.tests.chunk.MultiChunkerTest

org.syncany.tests.operations.AssemblerTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.