Package org.jets3t.service.multithread

Examples of org.jets3t.service.multithread.S3ServiceMulti


            }
                   
            // Upload New/Updated/Forced/Replaced objects to S3.
            if (doAction && objectsToUpload.size() > 0) {
                S3Object[] objects = (S3Object[]) objectsToUpload.toArray(new S3Object[objectsToUpload.size()]);
                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).putObjects(bucket, objects);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
                        throw new Exception(thrown);
                    }
                }
            }
        } while (priorLastKey != null);
       
        // Delete objects on S3 that don't correspond with local files.
        List objectsToDelete = new ArrayList();
        Iterator serverOnlyIter = mergedDiscrepancyResults.onlyOnServerKeys.iterator();
        while (serverOnlyIter.hasNext()) {
            // Relative key
            String relativeKeyPath = (String) serverOnlyIter.next();

            // Build absolute key path for object.
            String targetKey = relativeKeyPath;
            if (rootObjectPath.length() > 0) {
                if (rootObjectPath.endsWith(Constants.FILE_PATH_DELIM)) {
                    targetKey = rootObjectPath + targetKey;                        
                } else {
                    targetKey = rootObjectPath + Constants.FILE_PATH_DELIM + targetKey;                        
                }
            }
            S3Object s3Object = new S3Object(targetKey);
                       
            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + relativeKeyPath, REPORT_LEVEL_DIFFERENCES);               
            } else {
                printOutputLine("D " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(s3Object);
                }
            }
        }
        if (objectsToDelete.size() > 0) {
            S3Object[] objects = (S3Object[]) objectsToDelete.toArray(new S3Object[objectsToDelete.size()]);
            (new S3ServiceMulti(s3Service, serviceEventAdaptor)).deleteObjects(bucket, objects);
            if (serviceEventAdaptor.wasErrorThrown()) {
                Throwable thrown = serviceEventAdaptor.getErrorThrown();
                if (thrown instanceof Exception) {
                    throw (Exception) thrown;
                } else {
View Full Code Here


           
            // Download New/Updated/Forced/Replaced objects from S3.
            if (doAction && downloadPackagesList.size() > 0) {
                DownloadPackage[] downloadPackages = (DownloadPackage[])
                    downloadPackagesList.toArray(new DownloadPackage[downloadPackagesList.size()]);
                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).downloadObjects(bucket, downloadPackages);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
                        throw new Exception(thrown);
                    }
                }
            }
        } while (priorLastKey != null);                       

        // Delete local files that don't correspond with S3 objects.
        ArrayList dirsToDelete = new ArrayList();
        Iterator clientOnlyIter = mergedDiscrepancyResults.onlyOnClientKeys.iterator();
        while (clientOnlyIter.hasNext()) {
            String keyPath = (String) clientOnlyIter.next();
            File file = (File) filesMap.get(keyPath);
           
            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + keyPath, REPORT_LEVEL_DIFFERENCES);               
            } else {
                printOutputLine("D " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    if (file.isDirectory()) {
                        // Delete directories later, as they may still have files
                        // inside until this loop completes.
                        dirsToDelete.add(file);
                    } else {
                        file.delete();
                    }
                }
            }
        }
        Iterator dirIter = dirsToDelete.iterator();
        while (dirIter.hasNext()) {
            File dir = (File) dirIter.next();
            dir.delete();
        }
       
        // Delete objects in S3 that have been moved to the local computer.
        List objectsMoved = new ArrayList();
        if (isMoveEnabled) {
            objectsMoved.addAll(mergedDiscrepancyResults.onlyOnServerKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.updatedOnServerKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.updatedOnClientKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.alreadySynchronisedKeys);
            Collections.sort(objectsMoved);
           
            Iterator objectsMovedIter = objectsMoved.iterator();
           
            List objectsToDelete = new ArrayList();
            while (objectsMovedIter.hasNext()) {
                String keyPath = (String) objectsMovedIter.next();
                S3Object s3Object = new S3Object(keyPath);

                printOutputLine("M " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(s3Object);
                }
            }
           
            if (objectsToDelete.size() > 0) {
                S3Object[] objects = (S3Object[]) objectsToDelete.toArray(new S3Object[objectsToDelete.size()]);
                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).deleteObjects(bucket, objects);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
View Full Code Here

                this);

        // Initialise a non-authenticated service.
        try {
            // Revert to anonymous service.
            s3ServiceMulti = new S3ServiceMulti(
                new RestS3Service(null, APPLICATION_DESCRIPTION, this), this);
        } catch (S3ServiceException e) {
            String message = "Unable to start anonymous service";
            log.error(message, e);
            ErrorDialog.showDialog(ownerFrame, this, cockpitLiteProperties.getProperties(), message, e);
View Full Code Here

       
        /*
         * Create a S3ServiceMulti object with an event listener that responds to
         * ListObjectsEvent notifications and populates a complete object listing.
         */
        final S3ServiceMulti s3Multi = new S3ServiceMulti(s3Service, new S3ServiceEventAdaptor() {
            public void s3ServiceEventPerformed(ListObjectsEvent event) {
                if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                    Iterator chunkIter = event.getChunkList().iterator();
                    while (chunkIter.hasNext()) {
                        S3ObjectsChunk chunk = (S3ObjectsChunk) chunkIter.next();
                       
                        if (log.isDebugEnabled()) {
                          log.debug("Listed " + chunk.getObjects().length
                            + " objects and " + chunk.getCommonPrefixes().length
                            + " common prefixes in bucket '" + bucketName
                            + "' using prefix=" + chunk.getPrefix()
                            + ", delimiter=" + chunk.getDelimiter());
                        }

                        allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                    }
                } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                    s3ServiceExceptions[0] = new S3ServiceException(
                        "Failed to list all objects in S3 bucket",
                        event.getErrorCause());                   
                }
            }
        })

        // The first listing partition we use as a starting point is the target path.
        String[] prefixesToList = new String[] {targetPath};
        int currentDepth = 0;       
       
        while (currentDepth <= toDepth && prefixesToList.length > 0) {
          if (log.isDebugEnabled()) {
            log.debug("Listing objects in '" + bucketName + "' using "
                + prefixesToList.length + " prefixes: "
                + Arrays.asList(prefixesToList));
          }

            // Initialize the variables that will be used, or populated, by the
            // multi-threaded listing.
            lastCommonPrefixes.clear();
            final String[] finalPrefixes = prefixesToList;
            final String finalDelimiter = (currentDepth < toDepth ? delimiter : null);

            /*
             * Perform a multi-threaded listing, where each prefix string
             * will be used as a unique partition to be listed in a separate thread.
             */
            (new Thread() {
                public void run() {
                    s3Multi.listObjects(bucketName, finalPrefixes,
                        finalDelimiter, Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE);
                };
            }).run();
            // Throw any exceptions that occur inside the threads.
            if (s3ServiceExceptions[0] != null) {
View Full Code Here

            s3Objects = s3ObjectsIncomplete;           
        } else {
            // Retrieve the complete information about all objects listed via GetObjectsHeads.
            final ArrayList s3ObjectsCompleteList = new ArrayList(s3ObjectsIncomplete.length);
            final S3ServiceException s3ServiceExceptions[] = new S3ServiceException[1];
            S3ServiceMulti s3ServiceMulti = new S3ServiceMulti(s3Service, new S3ServiceEventAdaptor() {
                public void s3ServiceEventPerformed(GetObjectHeadsEvent event) {
                    if (GetObjectHeadsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        S3Object[] finishedObjects = event.getCompletedObjects();
                        if (finishedObjects.length > 0) {
                            s3ObjectsCompleteList.addAll(Arrays.asList(finishedObjects));
                        }
                    } else if (GetObjectHeadsEvent.EVENT_ERROR == event.getEventCode()) {
                        s3ServiceExceptions[0] = new S3ServiceException(
                            "Failed to retrieve detailed information about all S3 objects",
                            event.getErrorCause());                   
                    }
                }
            });
            if (s3ServiceEventListener != null) {
                s3ServiceMulti.addServiceEventListener(s3ServiceEventListener);
            }
            s3ServiceMulti.getObjectsHeads(bucket, s3ObjectsIncomplete);
            if (s3ServiceExceptions[0] != null) {
                throw s3ServiceExceptions[0];
            }       
            s3Objects = (S3Object[]) s3ObjectsCompleteList
                .toArray(new S3Object[s3ObjectsCompleteList.size()]);
View Full Code Here

                    LazyPreparedUploadObject lazyObj =
                        (LazyPreparedUploadObject) objectsToUpload.remove(0);
                    objects[i] = lazyObj.prepareUploadObject();
                }

                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).putObjects(bucket, objects);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
                        throw new Exception(thrown);
                    }
                }
                partialUploadObjectsProgressCount += objects.length;
            }
        } while (priorLastKey != null);
       
        // Delete objects on S3 that don't correspond with local files.
        List objectsToDelete = new ArrayList();
        Iterator serverOnlyIter = mergedDiscrepancyResults.onlyOnServerKeys.iterator();
        while (serverOnlyIter.hasNext()) {
            // Relative key
            String relativeKeyPath = (String) serverOnlyIter.next();

            // Build absolute key path for object.
            String targetKey = relativeKeyPath;
            if (rootObjectPath.length() > 0) {
                if (rootObjectPath.endsWith(Constants.FILE_PATH_DELIM)) {
                    targetKey = rootObjectPath + targetKey;                        
                } else {
                    targetKey = rootObjectPath + Constants.FILE_PATH_DELIM + targetKey;                        
                }
            }
            S3Object s3Object = new S3Object(targetKey);
                       
            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + relativeKeyPath, REPORT_LEVEL_DIFFERENCES);               
            } else {
                printOutputLine("D " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(s3Object);
                }
            }
        }
        if (objectsToDelete.size() > 0) {
            S3Object[] objects = (S3Object[]) objectsToDelete.toArray(new S3Object[objectsToDelete.size()]);
            (new S3ServiceMulti(s3Service, serviceEventAdaptor)).deleteObjects(bucket, objects);
            if (serviceEventAdaptor.wasErrorThrown()) {
                Throwable thrown = serviceEventAdaptor.getErrorThrown();
                if (thrown instanceof Exception) {
                    throw (Exception) thrown;
                } else {
View Full Code Here

           
            // Download New/Updated/Forced/Replaced objects from S3.
            if (doAction && downloadPackagesList.size() > 0) {
                DownloadPackage[] downloadPackages = (DownloadPackage[])
                    downloadPackagesList.toArray(new DownloadPackage[downloadPackagesList.size()]);
                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).downloadObjects(bucket, downloadPackages);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
                        throw new Exception(thrown);
                    }
                }
            }
        } while (priorLastKey != null);                       

        // Delete local files that don't correspond with S3 objects.
        ArrayList dirsToDelete = new ArrayList();
        Iterator clientOnlyIter = mergedDiscrepancyResults.onlyOnClientKeys.iterator();
        while (clientOnlyIter.hasNext()) {
            String keyPath = (String) clientOnlyIter.next();
            File file = (File) filesMap.get(keyPath);
           
            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + keyPath, REPORT_LEVEL_DIFFERENCES);               
            } else {
                printOutputLine("D " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    if (file.isDirectory()) {
                        // Delete directories later, as they may still have files
                        // inside until this loop completes.
                        dirsToDelete.add(file);
                    } else {
                        file.delete();
                    }
                }
            }
        }
        Iterator dirIter = dirsToDelete.iterator();
        while (dirIter.hasNext()) {
            File dir = (File) dirIter.next();
            dir.delete();
        }
       
        // Delete objects in S3 that have been moved to the local computer.
        List objectsMoved = new ArrayList();
        if (isMoveEnabled) {
            objectsMoved.addAll(mergedDiscrepancyResults.onlyOnServerKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.updatedOnServerKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.updatedOnClientKeys);
            objectsMoved.addAll(mergedDiscrepancyResults.alreadySynchronisedKeys);
            Collections.sort(objectsMoved);
           
            Iterator objectsMovedIter = objectsMoved.iterator();
           
            List objectsToDelete = new ArrayList();
            while (objectsMovedIter.hasNext()) {
                String keyPath = (String) objectsMovedIter.next();
                S3Object s3Object = new S3Object(keyPath);

                printOutputLine("M " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(s3Object);
                }
            }
           
            if (objectsToDelete.size() > 0) {
                S3Object[] objects = (S3Object[]) objectsToDelete.toArray(new S3Object[objectsToDelete.size()]);
                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).deleteObjects(bucket, objects);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
View Full Code Here

            SignedUrlAndObject[] uploadItems = prepareSignedObjects(
                objectsForUpload, gatekeeperMessage.getSignatureRequests(), xmlGenerator);

            if (s3ServiceMulti == null) {
                s3ServiceMulti = new S3ServiceMulti(
                    new RestS3Service(null, APPLICATION_DESCRIPTION, this), this);
            }

            /*
             * Prepare XML Summary document for upload, if the summary option is set.
View Full Code Here

                    log.debug("Log in cancelled by user");
                    return;
                }
            }

            s3ServiceMulti = new S3ServiceMulti(
                getRestS3Service(credentials), this);

            cloudFrontMembershipChecked = false;
            listAllBuckets();
View Full Code Here

                loginAwsCredentialsMap.remove(loginName);
                loginSwitchMenu.setEnabled(loginAwsCredentialsMap.size() > 0);
            }

            // Revert to anonymous service.
            s3ServiceMulti = new S3ServiceMulti(
                getRestS3Service(null), this);
            cloudFrontService = null;

            bucketsTable.clearSelection();
            bucketTableModel.removeAllBuckets();
View Full Code Here

TOP

Related Classes of org.jets3t.service.multithread.S3ServiceMulti

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.