Examples of org.jets3t.service.S3ObjectsChunk

org.jets3t.service.S3ObjectsChunk
Stores a "chunk" of S3Objects returned from a list command - this particular chunk may or may not include all the objects available in a bucket. This class contains an array of S3objects and a the last key name returned by a prior call to the method {@link S3Service#listObjectsChunked(String,String,String,long,String)}. @author James Murty

        final S3ServiceMulti s3Multi = new S3ServiceMulti(s3Service, new S3ServiceEventAdaptor() {
            public void s3ServiceEventPerformed(ListObjectsEvent event) {
                if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                    Iterator chunkIter = event.getChunkList().iterator();
                    while (chunkIter.hasNext()) {
                        S3ObjectsChunk chunk = (S3ObjectsChunk) chunkIter.next();
                        
                        if (log.isDebugEnabled()) {
                          log.debug("Listed " + chunk.getObjects().length
                            + " objects and " + chunk.getCommonPrefixes().length 
                            + " common prefixes in bucket '" + bucketName 
                            + "' using prefix=" + chunk.getPrefix() 
                            + ", delimiter=" + chunk.getDelimiter());
                        }


                        allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                    }
                } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                    s3ServiceExceptions[0] = new S3ServiceException(
                        "Failed to list all objects in S3 bucket", 
                        event.getErrorCause());

View Full Code Here

        S3Object[] objects = null;
        String resultPriorLastKey = null;
        if (completeListing) {
            objects = listObjectsThreaded(s3Service, bucket.getName(), prefix);
        } else {
            S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), 
                prefix, null, Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE, 
                priorLastKey, completeListing);
            objects = chunk.getObjects();
            resultPriorLastKey = chunk.getPriorLastKey();
        }


        Map objectsMap = buildS3ObjectMap(s3Service, bucket, targetPath, 
            objects, skipMetadata, s3ServiceEventListener);
        return new PartialObjectListing(objectsMap, resultPriorLastKey);

View Full Code Here

        }
        if (automaticallyMergeChunks) {
            if (log.isDebugEnabled()) {
              log.debug("Found " + objects.size() + " objects in total");
            }
            return new S3ObjectsChunk(
                prefix, delimiter, 
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                prefix, delimiter, 
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }

View Full Code Here

            try {   
                List allObjects = new ArrayList();
                List allCommonPrefixes = new ArrayList();
                
                do {                                
                    S3ObjectsChunk chunk = s3Service.listObjectsChunked(
                        bucketName, prefix, delimiter, maxListingLength, priorLastKey);                    
                    priorLastKey = chunk.getPriorLastKey();                
                    
                    allObjects.addAll(Arrays.asList(chunk.getObjects()));
                    allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                } while (!halted && priorLastKey != null);
                
                result = new S3ObjectsChunk(
                    prefix, delimiter,
                    (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]),
                    (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]),
                    null);
            } catch (S3ServiceException e) {

View Full Code Here

        }
        if (automaticallyMergeChunks) {
            if (log.isDebugEnabled()) {
                log.debug("Found " + objects.size() + " objects in total");
            }
            return new S3ObjectsChunk(
                prefix, delimiter,
                (S3Object[]) objects.toArray(new S3Object[objects.size()]),
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                prefix, delimiter,
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }

View Full Code Here

        final S3ServiceMulti s3Multi = new S3ServiceMulti(s3Service, new S3ServiceEventAdaptor() {
            public void s3ServiceEventPerformed(ListObjectsEvent event) {
                if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                    Iterator chunkIter = event.getChunkList().iterator();
                    while (chunkIter.hasNext()) {
                        S3ObjectsChunk chunk = (S3ObjectsChunk) chunkIter.next();
                        
                        if (log.isDebugEnabled()) {
                          log.debug("Listed " + chunk.getObjects().length
                            + " objects and " + chunk.getCommonPrefixes().length 
                            + " common prefixes in bucket '" + bucketName 
                            + "' using prefix=" + chunk.getPrefix() 
                            + ", delimiter=" + chunk.getDelimiter());
                        }


                        allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                    }
                } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                    s3ServiceExceptions[0] = new S3ServiceException(
                        "Failed to list all objects in S3 bucket", 
                        event.getErrorCause());

View Full Code Here

        S3Object[] objects = null;
        String resultPriorLastKey = null;
        if (completeListing) {
            objects = listObjectsThreaded(s3Service, bucket.getName(), prefix);
        } else {
            S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), 
                prefix, null, Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE, 
                priorLastKey, completeListing);
            objects = chunk.getObjects();
            resultPriorLastKey = chunk.getPriorLastKey();
        }


        Map objectsMap = buildS3ObjectMap(s3Service, bucket, targetPath, 
            objects, skipMetadata, s3ServiceEventListener);
        return new PartialObjectListing(objectsMap, resultPriorLastKey);

View Full Code Here

        }
        if (automaticallyMergeChunks) {
            if (log.isDebugEnabled()) {
                log.debug("Found " + objects.size() + " objects in total");
            }
            return new S3ObjectsChunk(
                prefix, delimiter,
                (S3Object[]) objects.toArray(new S3Object[objects.size()]),
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                null);
        } else {
            return new S3ObjectsChunk(
                prefix, delimiter,
                (S3Object[]) objects.toArray(new S3Object[objects.size()]), 
                (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
                priorLastKey);            
        }

View Full Code Here


                    allObjects.addAll(Arrays.asList(chunk.getObjects()));
                    allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                } while (!halted && priorLastKey != null);


                result = new S3ObjectsChunk(
                    prefix, delimiter,
                    (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]),
                    (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]),
                    null);
            } catch (ServiceException se) {

View Full Code Here

      int maxListingLength, String priorLastKey) throws IOException {
    try {
      if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
        prefix += PATH_DELIMITER;
      }
      S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
          prefix, delimiter, maxListingLength, priorLastKey);
      
      FileMetadata[] fileMetadata =
        new FileMetadata[chunk.getObjects().length];
      for (int i = 0; i < fileMetadata.length; i++) {
        S3Object object = chunk.getObjects()[i];
        fileMetadata[i] = new FileMetadata(object.getKey(),
            object.getContentLength(), object.getLastModifiedDate().getTime());
      }
      return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
          chunk.getCommonPrefixes());
    } catch (S3ServiceException e) {
      if (e.getCause() instanceof IOException) {
        throw (IOException) e.getCause();
      }
      throw new S3Exception(e);

View Full Code Here

0 1 2

TOP

Related Classes of org.jets3t.service.S3ObjectsChunk

org.apache.hadoop.fs.s3native.Jets3tNativeFileSystemStore

org.jets3t.apps.cockpit.Cockpit

org.jets3t.samples.ThreadedObjectListing

org.jets3t.service.impl.rest.httpclient.RestS3Service

org.jets3t.service.impl.soap.axis.SoapS3Service

org.jets3t.service.multithread.S3ServiceMulti$ListObjectsRunnable

org.jets3t.service.utils.FileComparer

org.jets3t.tests.BaseS3ServiceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.