Package org.jets3t.service.multi

Examples of org.jets3t.service.multi.ThreadedStorageService$DownloadObjectRunnable


                        }
                    }

                    // Perform standard object uploads
                    if (objectsForStandardPut.size() > 0) {
                        (new ThreadedStorageService(storageService, serviceEventAdaptor)).putObjects(
                            bucket.getName(), objectsForStandardPut.toArray(new StorageObject[] {}));
                        serviceEventAdaptor.throwErrorIfPresent();
                    }

                    // Perform multipart uploads
                    if (objectsForMultipartUpload.size() > 0) {
                        multipartUtils.uploadObjects(
                            bucket.getName(), (S3Service)storageService,
                            objectsForMultipartUpload, serviceEventAdaptor);
                    }
                }
            } while (objectKeyIter.hasNext()); // End of upload loop

        } while (priorLastKey != null); // End of list and upload loop

        // Delete objects that don't correspond with local files.
        List<StorageObject> objectsToDelete = new ArrayList<StorageObject>();
        Iterator<String> serverOnlyIter = mergedDiscrepancyResults.onlyOnServerKeys.iterator();
        while (serverOnlyIter.hasNext()) {
            // Relative key
            String relativeKeyPath = serverOnlyIter.next();

            // Build absolute key path for object.
            String targetKey = relativeKeyPath;
            if (rootObjectPath.length() > 0) {
                if (rootObjectPath.endsWith(Constants.FILE_PATH_DELIM)) {
                    targetKey = rootObjectPath + targetKey;
                } else {
                    targetKey = rootObjectPath + Constants.FILE_PATH_DELIM + targetKey;
                }
            }
            StorageObject object = new StorageObject(targetKey);

            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + relativeKeyPath, REPORT_LEVEL_DIFFERENCES);
            } else {
                printOutputLine("D " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(object);
                }
            }
        }
        if (objectsToDelete.size() > 0) {
            StorageObject[] objects = objectsToDelete.toArray(new StorageObject[objectsToDelete.size()]);
            (new ThreadedStorageService(storageService, serviceEventAdaptor)).deleteObjects(bucket.getName(), objects);
            serviceEventAdaptor.throwErrorIfPresent();
        }

        // Delete local files that have been moved to service.
        List<String> filesMoved = new ArrayList<String>();
View Full Code Here


                // Download New/Updated/Forced/Replaced objects from service.
                if (doAction && downloadPackagesList.size() > 0) {
                    DownloadPackage[] downloadPackages = downloadPackagesList.toArray(
                        new DownloadPackage[downloadPackagesList.size()]);
                    (new ThreadedStorageService(storageService, serviceEventAdaptor)).downloadObjects(
                        bucket.getName(), downloadPackages);
                    serviceEventAdaptor.throwErrorIfPresent();
                }
            } while (objectKeyIter.hasNext());

        } while (priorLastKey != null);

        // Delete local files that don't correspond with service objects.
        List<File> dirsToDelete = new ArrayList<File>();
        Iterator<String> clientOnlyIter = mergedDiscrepancyResults.onlyOnClientKeys.iterator();
        while (clientOnlyIter.hasNext()) {
            String keyPath = clientOnlyIter.next();
            File file = new File(objectKeyToFilepathMap.get(keyPath));

            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + keyPath, REPORT_LEVEL_DIFFERENCES);
            } else {
                printOutputLine("D " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    if (file.isDirectory()) {
                        // Delete directories later, as they may still have files
                        // inside until this loop completes.
                        dirsToDelete.add(file);
                    } else {
                        file.delete();
                    }
                }
            }
        }
        Iterator<File> dirIter = dirsToDelete.iterator();
        while (dirIter.hasNext()) {
            File dir = dirIter.next();
            dir.delete();
        }

        // Delete objects in service that have been moved to the local computer.
        if (isMoveEnabled) {
            List<String> objectsMovedLocalPaths = new ArrayList<String>(objectsMoved.size());
            objectsMovedLocalPaths.addAll(objectsMoved.keySet());
            Collections.sort(objectsMovedLocalPaths);

            for (String movedLocalPath: objectsMovedLocalPaths) {
                if (doAction) {
                    printOutputLine("M " + movedLocalPath, REPORT_LEVEL_ACTIONS);
                } else {
                    printOutputLine("m " + movedLocalPath, REPORT_LEVEL_ACTIONS);
                }
            }

            if (objectsMoved.size() > 0 && doAction) {
                StorageObject[] objects = objectsMoved.values().toArray(
                    new StorageObject[objectsMoved.size()]);
                (new ThreadedStorageService(storageService, serviceEventAdaptor)).deleteObjects(
                    bucket.getName(), objects);
                serviceEventAdaptor.throwErrorIfPresent();
            }
        }
View Full Code Here

            final int[] getObjectHeadsEventCount = new int[] {0};
            final List<StorageObject> getObjectsList = new ArrayList<StorageObject>();
            final int[] deleteObjectsEventCount = new int[] {0};

            // Multi-threaded service with adaptor to count event occurrences.
            ThreadedStorageService threadedService = new ThreadedStorageService(
                service,
                new StorageServiceEventAdaptor() {
                    @Override
                    public void event(CreateObjectsEvent event) {
                        if (CreateObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                            createObjectsEventCount[0] += event.getCreatedObjects().length;
                        }
                    }

                    @Override
                    public void event(GetObjectHeadsEvent event) {
                        if (GetObjectHeadsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                            getObjectHeadsEventCount[0] += event.getCompletedObjects().length;
                        }
                    }

                    @Override
                    public void event(GetObjectsEvent event) {
                        if (GetObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                            for (StorageObject object: event.getCompletedObjects()) {
                                getObjectsList.add(object);
                            }
                        }
                    }

                    @Override
                    public void event(DeleteObjectsEvent event) {
                        if (DeleteObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                            deleteObjectsEventCount[0] += event.getDeletedObjects().length;
                        }
                    }
            });

            assertEquals(0, createObjectsEventCount[0]);
            assertEquals(0, getObjectHeadsEventCount[0]);
            assertEquals(0, getObjectsList.size());
            assertEquals(0, deleteObjectsEventCount[0]);

            StorageObject[] objects = new StorageObject[] {
                new StorageObject("one.txt", "Some data"),
                new StorageObject("twö.txt", "Some data"),
                new StorageObject("thréè.txt", "Some data"),
                new StorageObject("fôür.txt", "Some data"),
                new StorageObject("fîvæ∫.txt", "Some data")
            };

            // Upload multiple objects
            boolean success = threadedService.putObjects(bucketName, objects);
            assertTrue(success);
            assertEquals(objects.length, createObjectsEventCount[0]);
            assertEquals(0, getObjectHeadsEventCount[0]);
            assertEquals(0, getObjectsList.size());
            assertEquals(0, deleteObjectsEventCount[0]);

            // Retrieve details for multiple objects
            success = threadedService.getObjectsHeads(bucketName, objects);
            assertTrue(success);
            assertEquals(objects.length, createObjectsEventCount[0]);
            assertEquals(objects.length, getObjectHeadsEventCount[0]);
            assertEquals(0, getObjectsList.size());
            assertEquals(0, deleteObjectsEventCount[0]);

            // Retrieve data for multiple objects
            success = threadedService.getObjects(bucketName, objects);
            assertTrue(success);
            assertEquals(objects.length, createObjectsEventCount[0]);
            assertEquals(objects.length, getObjectHeadsEventCount[0]);
            assertEquals(objects.length, getObjectsList.size());
            assertEquals(0, deleteObjectsEventCount[0]);
            // Check all objects retrieved have expected data content.
            for (StorageObject getObject: getObjectsList) {
                // TODO: Google Storage doesn't reliably return Content-Length in a GET
                if (!TARGET_SERVICE_GS.equals(getTargetService())) {
                    assertEquals("Some data".length(), getObject.getContentLength());
                }
                String objectData = ServiceUtils.readInputStreamToString(
                    getObject.getDataInputStream(), Constants.DEFAULT_ENCODING);
                assertEquals("Some data", objectData);
            }

            // Delete multiple objects
            success = threadedService.deleteObjects(bucketName, objects);
            assertTrue(success);
            assertEquals(objects.length, createObjectsEventCount[0]);
            assertEquals(objects.length, getObjectHeadsEventCount[0]);
            assertEquals(objects.length, getObjectsList.size());
            assertEquals(objects.length, deleteObjectsEventCount[0]);
View Full Code Here

        /*
         * Create a ThreadedStorageService object with an event listener that responds to
         * ListObjectsEvent notifications and populates a complete object listing.
         */
        final ThreadedStorageService threadedService = new ThreadedStorageService(service,
            new StorageServiceEventAdaptor() {
            @Override
            public void event(ListObjectsEvent event) {
                if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                    Iterator<StorageObjectsChunk> chunkIter = event.getChunkList().iterator();
                    while (chunkIter.hasNext()) {
                        StorageObjectsChunk chunk = chunkIter.next();

                        if (log.isDebugEnabled()) {
                            log.debug("Listed " + chunk.getObjects().length
                            + " objects and " + chunk.getCommonPrefixes().length
                            + " common prefixes in bucket '" + bucketName
                            + "' using prefix=" + chunk.getPrefix()
                            + ", delimiter=" + chunk.getDelimiter());
                        }

                        allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                    }
                } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                    serviceExceptions[0] = new ServiceException(
                        "Failed to list all objects in bucket",
                        event.getErrorCause());
                }
            }
        });

        // The first listing partition we use as a starting point is the target path.
        String[] prefixesToList = new String[] {targetPath};
        int currentDepth = 0;

        while (currentDepth <= toDepth && prefixesToList.length > 0) {
            if (log.isDebugEnabled()) {
                log.debug("Listing objects in '" + bucketName + "' using "
                + prefixesToList.length + " prefixes: "
                + Arrays.asList(prefixesToList));
            }

            // Initialize the variables that will be used, or populated, by the
            // multi-threaded listing.
            lastCommonPrefixes.clear();
            final String[] finalPrefixes = prefixesToList;
            final String finalDelimiter = (currentDepth < toDepth ? delimiter : null);

            /*
             * Perform a multi-threaded listing, where each prefix string
             * will be used as a unique partition to be listed in a separate thread.
             */
            (new Thread() {
                @Override
                public void run() {
                    threadedService.listObjects(bucketName, finalPrefixes,
                        finalDelimiter, Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE);
                };
            }).run();
            // Throw any exceptions that occur inside the threads.
            if (serviceExceptions[0] != null) {
View Full Code Here

        if (objectsForMetadataRetrieval.size() > 0) {
            // Retrieve the complete metadata information for selected objects
            final List<StorageObject> objectsCompleteList =
                new ArrayList<StorageObject>(objectsWithoutMetadata.length);
            final ServiceException serviceExceptions[] = new ServiceException[1];
            ThreadedStorageService threadedService = new ThreadedStorageService(service,
                new StorageServiceEventAdaptor() {
                @Override
                public void event(GetObjectHeadsEvent event) {
                    if (GetObjectHeadsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        StorageObject[] finishedObjects = event.getCompletedObjects();
                        if (finishedObjects.length > 0) {
                            objectsCompleteList.addAll(Arrays.asList(finishedObjects));
                        }
                    } else if (GetObjectHeadsEvent.EVENT_ERROR == event.getEventCode()) {
                        serviceExceptions[0] = new ServiceException(
                            "Failed to retrieve detailed information about all objects",
                            event.getErrorCause());
                    }
                }
            });
            if (eventListener != null) {
                threadedService.addServiceEventListener(eventListener);
            }
            threadedService.getObjectsHeads(bucketName,
                objectsForMetadataRetrieval.toArray(new StorageObject[] {}));
            if (serviceExceptions[0] != null) {
                throw serviceExceptions[0];
            }
View Full Code Here

TOP

Related Classes of org.jets3t.service.multi.ThreadedStorageService$DownloadObjectRunnable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.