Examples of com.google.appengine.tools.cloudstorage.GcsService

Package com.google.appengine.tools.cloudstorage

Examples of com.google.appengine.tools.cloudstorage.GcsService

com.google.appengine.tools.cloudstorage.GcsService
Allows creating and accessing files in Google Cloud Storage. This class uses the {@link RetryParams} that were passed to {@link GcsServiceFactory}. This is used for all of the methods provided by this class. So an {@link RetriesExhaustedException} meansthat these have failed. Retry logic is handled internally to this class, because while writing to the {@link GcsOutputChannel} a request can span the boundaries of segments written to{@link RawGcsService}. As a result a write call could partly succeed, making error recovery at an application level next to impossible. Reading, deleting, and gettingMetadata are all idempotent operations. However for symmetry and convenience the {@link RetryParams} are applied to these calls as well.

    super.setUp();
    helper.setUp();
  }


  long createFile(GcsFilename filename, String record, int recordsCount) throws IOException {
    GcsService gcsService = GcsServiceFactory.createGcsService();
    try (GcsOutputChannel writeChannel = gcsService.createOrReplace(
        filename, new GcsFileOptions.Builder().mimeType("application/bin").build())) {
      for (int i = 0; i < recordsCount; i++) {
        writeChannel.write(ByteBuffer.wrap(record.getBytes()));
      }
    }
    return gcsService.getMetadata(filename).getLength();
  }

View Full Code Here

      @Override
      public void verify(MapReduceResult<GoogleCloudStorageFileSet> result) throws Exception {
        assertEquals(2, result.getOutputResult().getNumFiles());
        assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue());
        ArrayList<Long> results = new ArrayList<>();
        GcsService gcsService = GcsServiceFactory.createGcsService();
        ByteBuffer holder = ByteBuffer.allocate(8);
        for (GcsFilename file : result.getOutputResult().getFiles()) {
          try (GcsInputChannel channel = gcsService.openPrefetchingReadChannel(file, 0, 4096)) {
            int read = channel.read(holder);
            while (read != -1) {
              holder.rewind();
              results.add(holder.getLong());
              holder.rewind();

View Full Code Here


  @Override
  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    resp.setContentType("text/plain");
    resp.getWriter().println("Hello, world from java");
    GcsService gcsService = GcsServiceFactory.createGcsService();
    GcsFilename filename = new GcsFilename(BUCKETNAME, FILENAME);
    GcsFileOptions options = new GcsFileOptions.Builder()
        .mimeType("text/html")
        .acl("public-read")
        .addUserMetadata("myfield1", "my field value")
        .build();


    GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
    PrintWriter writer = new PrintWriter(Channels.newWriter(writeChannel, "UTF8"));
    writer.println("The woods are lovely dark and deep.");
    writer.println("But I have promises to keep.");
    writer.flush();


    writeChannel.waitForOutstandingWrites();


    writeChannel.write(ByteBuffer.wrap("And miles to go before I sleep.".getBytes("UTF8")));


    writeChannel.close();
    resp.getWriter().println("Done writing...");




    GcsInputChannel readChannel = null;
    BufferedReader reader = null;
    try {
      readChannel = gcsService.openReadChannel(filename, 0);
      reader = new BufferedReader(Channels.newReader(readChannel, "UTF8"));
      String line;
      while ((line = reader.readLine()) != null) {
        resp.getWriter().println("READ:" + line);
      }

View Full Code Here

  private List<List<GcsFilename>> bundleFiles(List<GcsFilename> files, long bundleSizeLimit)
      throws IOException {
    List<List<GcsFilename>> bundles = new ArrayList<>();
    List<GcsFilename> currentBundle = new ArrayList<>();
    long currentBundleSize = 0;
    GcsService GCS_SERVICE = GcsServiceFactory.createGcsService();
    for (GcsFilename file : files) {
      long fileSize = GCS_SERVICE.getMetadata(file).getLength();
      if (currentBundleSize + fileSize > bundleSizeLimit) {
        bundles.add(ImmutableSet.copyOf(currentBundle).asList());
        currentBundle = new ArrayList<>();
        currentBundleSize = 0;
      }

View Full Code Here

    this.bufferSize = bufferSize;
  }


  @Override
  public List<? extends InputReader<byte[]>> createReaders() {
    GcsService gcsService =
        GcsServiceFactory.createGcsService(MapReduceConstants.GCS_RETRY_PARAMETERS);
    GcsFileMetadata metadata;
    try {
      metadata = gcsService.getMetadata(file);
      if (metadata == null) {
        throw new RuntimeException("File does not exist: " + file);
      }
    } catch (IOException e) {
      throw new RuntimeException("Unable to read file metadata: " + file, e);

View Full Code Here

    }
    return bucket;
  }


  private static void verifyBucketIsWritable(String bucket) throws IOException {
    GcsService gcsService = GcsServiceFactory.createGcsService(new RetryParams.Builder()
        .retryMinAttempts(2)
        .retryMaxAttempts(3)
        .totalRetryPeriodMillis(20000)
        .requestTimeoutMillis(10000)
        .build());
    GcsFilename filename = new GcsFilename(bucket, UUID.randomUUID() + ".tmp");
    if (gcsService.getMetadata(filename) != null) {
      log.warning("File '" + filename.getObjectName() + "' exists. Skipping bucket write test.");
      return;
    }
    try {
      gcsService.createOrReplace(filename, GcsFileOptions.getDefaultInstance(),
          ByteBuffer.wrap("Delete me!".getBytes(StandardCharsets.UTF_8)));
    } finally {
      gcsService.delete(filename);
    }
  }

View Full Code Here


        if (!uploadedFilename.equals(request.getParameter("uploadedFilename"))) {
            throw new IllegalStateException("Uploaded filename is incorrect: expecting " + request.getParameter("uploadedFilename") + " but got " + uploadedFilename);
        }


        GcsService service = GcsServiceFactory.createGcsService();


        String mimeType;
        ServletOutputStream outputStream = response.getOutputStream();


        if (gsObjectName.contains("fake")) { //GCS Local File Upload Implementation is broken
            log.warning("Using hacked blobstore-gcs hybrid test because the local gcs implementation is broken");
            String blobKeyString = gsObjectName.substring(gsObjectName.indexOf("-") + 1, gsObjectName.lastIndexOf("-"));
            DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService();
            BlobInfoFactory blobInfoFactory = new BlobInfoFactory(datastoreService);
            mimeType = blobInfoFactory.loadBlobInfo(new BlobKey(blobKeyString)).getContentType();


            ByteArrayInputStream bais = new ByteArrayInputStream(BlobstoreServiceFactory.getBlobstoreService().fetchData(new BlobKey(blobKeyString), 0, BlobstoreService.MAX_BLOB_FETCH_SIZE - 1));
            IOUtils.copyStream(bais, outputStream);
            outputStream.write("_123".getBytes());
        } else {
            GcsFilename filename = new GcsFilename("GcsBucket", gsObjectName);
            GcsFileMetadata metadata = service.getMetadata(filename);
            if (metadata == null) {
                throw new IllegalStateException("Null GCS metadata: " + filename);
            }
            mimeType = metadata.getOptions().getMimeType();


            try (InputStream inputStream = Channels.newInputStream(service.openReadChannel(metadata.getFilename(), 0))) {
                IOUtils.copyStream(inputStream, outputStream);
            }
            outputStream.write("_123".getBytes());
        }

View Full Code Here

    @InSequence(100)
    public void testCreateGsBlobKey() throws Exception {
        final long ts = System.currentTimeMillis();
        final byte[] bytes = "FooBar".getBytes();


        GcsService service = GcsServiceFactory.createGcsService();
        GcsFilename filename = new GcsFilename("GcsBucket", String.valueOf(ts));
        GcsFileOptions options = new GcsFileOptions.Builder().mimeType(CONTENT_TYPE).build();
        try (GcsOutputChannel out = service.createOrReplace(filename, options)) {
            IOUtils.copy(Channels.newChannel(new ByteArrayInputStream(bytes)), out);
        }


        BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
        BlobKey key =  blobstoreService.createGsBlobKey("/gs/GcsBucket/" + ts);

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.cloudstorage.GcsService

com.google.appengine.demos.PortOfFilesAPIGuestbookServlet

com.google.appengine.tck.blobstore.GcsBlobstoreUploadTest

com.google.appengine.tck.blobstore.support.GcsHandlerServlet

com.google.appengine.tools.mapreduce.bigqueryjobs.BigQueryLoadGoogleCloudStorageFilesJob

com.google.appengine.tools.mapreduce.EndToEndTest

com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLineInput

com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLineInputTestCase

com.google.appengine.tools.mapreduce.MapReduceSettings

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.