Examples of org.jets3t.service.S3ObjectsChunk

org.jets3t.service.S3ObjectsChunk
Stores a "chunk" of S3Objects returned from a list command - this particular chunk may or may not include all the objects available in a bucket. This class contains an array of S3objects and a the last key name returned by a prior call to the method {@link S3Service#listObjectsChunked(String,String,String,long,String)}. @author James Murty

      int maxListingLength, String priorLastKey) throws IOException {
    try {
      if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
        prefix += PATH_DELIMITER;
      }
      S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
          prefix, delimiter, maxListingLength, priorLastKey);
      
      FileMetadata[] fileMetadata =
        new FileMetadata[chunk.getObjects().length];
      for (int i = 0; i < fileMetadata.length; i++) {
        S3Object object = chunk.getObjects()[i];
        fileMetadata[i] = new FileMetadata(object.getKey(),
            object.getContentLength(), object.getLastModifiedDate().getTime());
      }
      return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
          chunk.getCommonPrefixes());
    } catch (S3ServiceException e) {
      if (e.getCause() instanceof IOException) {
        throw (IOException) e.getCause();
      }
      throw new S3Exception(e);

View Full Code Here

                    final String delimiter = (String) filterObjectsDelimiter.getSelectedItem();


                    final ArrayList allObjects = new ArrayList();
                    String priorLastKey = null;
                    do {
                        S3ObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked(
                            getCurrentSelectedBucket().getName(), prefix, delimiter, 
                            BUCKET_LIST_CHUNKING_SIZE, priorLastKey);
                        
                        final S3Object[] objects = chunk.getObjects();
                        for (int i = 0; i < objects.length; i++) {
                            objects[i].setOwner(getCurrentSelectedBucket().getOwner());
                        }                        
                        
                        priorLastKey = chunk.getPriorLastKey();
                        allObjects.addAll(Arrays.asList(objects));


                        updateProgressDialog(
                            "Listed " + allObjects.size() + " objects in " 
                            + getCurrentSelectedBucket().getName(), "", 0);

View Full Code Here

        } catch (Exception e) {
            throw new S3ServiceException("Unable to List Objects in bucket: " + bucketName, e);   
        }
        if (automaticallyMergeChunks) {
            log.debug("Found " + objects.size() + " objects in total");
            return new S3ObjectsChunk(
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }
    }

View Full Code Here

            if (!automaticallyMergeChunks)
                break;
        }
        if (automaticallyMergeChunks) {
            log.debug("Found " + objects.size() + " objects in total");
            return new S3ObjectsChunk(
                (S3Object[]) objects.toArray(new S3Object[objects.size()]),
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }
    }

View Full Code Here

        // List items in chunks of size 2, ensure we get a total of seven.
        int chunkedObjectsCount = 0;
        int chunkedIterationsCount = 0;
        String priorLastKey = null;
        do {
            S3ObjectsChunk chunk = s3Service.listObjectsChunked(
                bucket.getName(), null, null, 2, priorLastKey);
            priorLastKey = chunk.getPriorLastKey();
            chunkedObjectsCount += chunk.getObjects().length;
            chunkedIterationsCount++;
        } while (priorLastKey != null);
        assertEquals("Chunked bucket listing retreived incorrect number of objects", 
            objectsList.size(), chunkedObjectsCount);
        assertEquals("Chunked bucket listing ran for an unexpected number of iterations", 
            (objectsList.size() + 1) / 2, chunkedIterationsCount);
        
        // List objects with a prefix and delimiter to check common prefixes.
        S3ObjectsChunk chunk = s3Service.listObjectsChunked(
            bucket.getName(), "dir1/", "/", 100, null);
        assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of objects", 
            3, chunk.getObjects().length);
        assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of common prefixes", 
            1, chunk.getCommonPrefixes().length);
        
        // List the same items with a prefix.
        objects = s3Service.listObjects(bucket, "dir1", null);        
        assertEquals("Incorrect number of objects matching prefix", 7, objects.length);

View Full Code Here

        }
        if (automaticallyMergeChunks) {
            if (log.isDebugEnabled()) {
              log.debug("Found " + objects.size() + " objects in total");
            }
            return new S3ObjectsChunk(
                prefix, delimiter, 
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                prefix, delimiter, 
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }

View Full Code Here

            try {   
                List allObjects = new ArrayList();
                List allCommonPrefixes = new ArrayList();
                
                do {                                
                    S3ObjectsChunk chunk = s3Service.listObjectsChunked(
                        bucketName, prefix, delimiter, maxListingLength, priorLastKey);                    
                    priorLastKey = chunk.getPriorLastKey();                
                    
                    allObjects.addAll(Arrays.asList(chunk.getObjects()));
                    allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                } while (!halted && priorLastKey != null);
                
                result = new S3ObjectsChunk(
                    prefix, delimiter,
                    (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]),
                    (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]),
                    null);
            } catch (S3ServiceException e) {

View Full Code Here

                    final String delimiter = (String) filterObjectsDelimiter.getSelectedItem();


                    final ArrayList allObjects = new ArrayList();
                    String priorLastKey = null;
                    do {
                        S3ObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked(
                            getCurrentSelectedBucket().getName(), prefix, delimiter, 
                            BUCKET_LIST_CHUNKING_SIZE, priorLastKey);
                        
                        final S3Object[] objects = chunk.getObjects();
                        for (int i = 0; i < objects.length; i++) {
                            objects[i].setOwner(getCurrentSelectedBucket().getOwner());
                        }                        
                        
                        priorLastKey = chunk.getPriorLastKey();
                        allObjects.addAll(Arrays.asList(objects));


                        updateProgressDialog(
                            "Listed " + allObjects.size() + " objects in " 
                            + getCurrentSelectedBucket().getName(), "", 0);

View Full Code Here

                    final String delimiter = (String) filterObjectsDelimiter.getSelectedItem();


                    final ArrayList allObjects = new ArrayList();
                    String priorLastKey = null;
                    do {
                        S3ObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked(
                            currentSelectedBucket.getName(), prefix, delimiter, 
                            BUCKET_LIST_CHUNKING_SIZE, priorLastKey);
                        
                        final S3Object[] objects = chunk.getObjects();
                        for (int i = 0; i < objects.length; i++) {
                            objects[i].setOwner(currentSelectedBucket.getOwner());
                        }                        
                        
                        priorLastKey = chunk.getPriorLastKey();
                        allObjects.addAll(Arrays.asList(objects));


                        updateProgressDialog(
                            "Listed " + allObjects.size() + " objects in " 
                            + currentSelectedBucket.getName(), "", 0);

View Full Code Here

         * standard object listing with a delimiter string.  
         */
        long startTime = System.currentTimeMillis();
        
        // Find all the objects and common prefixes at the top level.
        S3ObjectsChunk initialChunk = restService.listObjectsChunked(bucketName, null, delimiter, 1000, null, true);
        
        long totalElapsedTime = System.currentTimeMillis() - startTime;
                
        // We will use the common prefix strings, if any, to perform sub-listings
        final String[] commonPrefixes = initialChunk.getCommonPrefixes();


        if (commonPrefixes.length > 0) {
            System.out.println("Performing sub-listings for common prefixes: " 
                + Arrays.asList(commonPrefixes));


            /*
             * Create a S3ServiceMulti object with an event listener that responds to
             * ListObjectsEvent notifications and populates a complete object listing. 
             */
            final S3ServiceMulti s3Multi = new S3ServiceMulti(restService, new S3ServiceEventAdaptor() {
                public void s3ServiceEventPerformed(ListObjectsEvent event) {
                    if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        Iterator chunkIter = event.getChunkList().iterator();
                        while (chunkIter.hasNext()) {
                            S3ObjectsChunk chunk = (S3ObjectsChunk) chunkIter.next();
                            
                            System.out.println("Listed " + chunk.getObjects().length 
                                + " objects for sub-listing with prefix: '" 
                                + chunk.getPrefix() + "'");
                            
                            allObjects.addAll(Arrays.asList(chunk.getObjects()));                            
                        }
                    } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                        s3ServiceExceptions[0] = new S3ServiceException(
                            "Failed to list all objects in S3 bucket", 
                            event.getErrorCause());

View Full Code Here

0 1 2

TOP

Related Classes of org.jets3t.service.S3ObjectsChunk

org.apache.hadoop.fs.s3native.Jets3tNativeFileSystemStore

org.jets3t.apps.cockpit.Cockpit

org.jets3t.samples.ThreadedObjectListing

org.jets3t.service.impl.rest.httpclient.RestS3Service

org.jets3t.service.impl.soap.axis.SoapS3Service

org.jets3t.service.multithread.S3ServiceMulti$ListObjectsRunnable

org.jets3t.service.utils.FileComparer

org.jets3t.tests.BaseS3ServiceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.