Examples of org.apache.twill.filesystem.Location

org.apache.twill.filesystem.Location
This interface defines the location and operations of a resource on the filesystem.
{@link Location} is agnostic to the type of file system the resource is on.

   *
   * @param config configuration of the stream
   * @return the generation id
   */
  public static int getGeneration(StreamConfig config) throws IOException {
    Location streamLocation = config.getLocation();


    // Default generation is 0.
    int genId = 0;
    CharMatcher numMatcher = CharMatcher.inRange('0', '9');


    List<Location> locations = streamLocation.list();
    if (locations == null) {
      return 0;
    }


    for (Location location : locations) {

View Full Code Here

          }
        }


        // Copy jar file into HDFS
        // Target path is the jarDir + jarMD5.jar
        final Location targetPath = jarDir.append("coprocessor" + hasher.hash().toString() + ".jar");


        // If the file exists and having same since, assume the file doesn't changed
        if (targetPath.exists() && targetPath.length() == jarFile.length()) {
          return targetPath;
        }


        // Copy jar file into filesystem
        if (!jarDir.mkdirs() && !jarDir.exists()) {
          throw new IOException("Fails to create directory: " + jarDir.toURI());
        }
        Files.copy(jarFile, new OutputSupplier<OutputStream>() {
        @Override
        public OutputStream getOutput() throws IOException {
          return targetPath.getOutputStream();
        }
      });
        return targetPath;
      } finally {
        jarFile.delete();

View Full Code Here

  @Rule
  public TemporaryFolder tmpDir = new TemporaryFolder();


  @Test
  public void testFindDependencies() throws IOException, ClassNotFoundException {
    Location location = new LocalLocationFactory(tmpDir.newFolder()).create("test.jar");


    // Create a jar file with by tracing dependency
    ApplicationBundler bundler = new ApplicationBundler(ImmutableList.<String>of());
    bundler.createBundle(location, ApplicationBundler.class);


    File targetDir = tmpDir.newFolder();
    unjar(new File(location.toURI()), targetDir);


    // Load the class back, it should be loaded by the custom classloader
    ClassLoader classLoader = createClassLoader(targetDir);
    Class<?> clz = classLoader.loadClass(ApplicationBundler.class.getName());
    Assert.assertSame(classLoader, clz.getClassLoader());

View Full Code Here

      }
    }
  }


  private void updateCredentials(String application, RunId runId, Credentials updates) throws IOException {
    Location credentialsLocation = locationFactory.create(String.format("/%s/%s/%s", application, runId.getId(),
                                                                        Constants.Files.CREDENTIALS));
    // Try to read the old credentials.
    Credentials credentials = new Credentials();
    if (credentialsLocation.exists()) {
      DataInputStream is = new DataInputStream(new BufferedInputStream(credentialsLocation.getInputStream()));
      try {
        credentials.readTokenStorageStream(is);
      } finally {
        is.close();
      }
    }


    // Overwrite with the updates.
    credentials.addAll(updates);


    // Overwrite the credentials.
    Location tmpLocation = credentialsLocation.getTempFile(Constants.Files.CREDENTIALS);


    // Save the credentials store with user-only permission.
    DataOutputStream os = new DataOutputStream(new BufferedOutputStream(tmpLocation.getOutputStream("600")));
    try {
      credentials.writeTokenStorageToStream(os);
    } finally {
      os.close();
    }


    // Rename the tmp file into the credentials location
    tmpLocation.renameTo(credentialsLocation);


    LOG.debug("Secure store for {} {} saved to {}.", application, runId, credentialsLocation.toURI());
  }

View Full Code Here

      return true;
    }


    try {
      Credentials credentials = new Credentials();
      Location location = getSecureStoreLocation();
      DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()));
      try {
        credentials.readTokenStorageStream(input);
      } finally {
        input.close();
      }


      UserGroupInformation.getCurrentUser().addCredentials(credentials);
      this.credentials = credentials;


      LOG.info("Secure store updated from {}.", location.toURI());


    } catch (Throwable t) {
      LOG.error("Failed to update secure store.", t);
    }

View Full Code Here

  }


  private void createAppMasterJar(ApplicationBundler bundler, Map<String, LocalFile> localFiles) throws IOException {
    try {
      LOG.debug("Create and copy {}", Constants.Files.APP_MASTER_JAR);
      Location location = createTempLocation(Constants.Files.APP_MASTER_JAR);


      List<Class<?>> classes = Lists.newArrayList();
      classes.add(ApplicationMasterMain.class);


      // Stuck in the yarnAppClient class to make bundler being able to pickup the right yarn-client version

View Full Code Here

      for (RuntimeSpecification spec : twillSpec.getRunnables().values()) {
        classes.add(classLoader.loadClass(spec.getRunnableSpecification().getClassName()));
      }


      LOG.debug("Create and copy {}", Constants.Files.CONTAINER_JAR);
      Location location = createTempLocation(Constants.Files.CONTAINER_JAR);
      bundler.createBundle(location, classes, resources);
      LOG.debug("Done {}", Constants.Files.CONTAINER_JAR);


      localFiles.put(Constants.Files.CONTAINER_JAR, createLocalFile(Constants.Files.CONTAINER_JAR, location));

View Full Code Here


    LOG.debug("Populating Runnable LocalFiles");
    for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) {
      String runnableName = entry.getKey();
      for (LocalFile localFile : entry.getValue().getLocalFiles()) {
        Location location;


        URI uri = localFile.getURI();
        if ("hdfs".equals(uri.getScheme())) {
          // Assuming the location factory is HDFS one. If it is not, it will failed, which is the correct behavior.
          location = locationFactory.create(uri);
        } else {
          URL url = uri.toURL();
          LOG.debug("Create and copy {} : {}", runnableName, url);
          // Preserves original suffix for expansion.
          location = copyFromURL(url, createTempLocation(Paths.appendSuffix(url.getFile(), localFile.getName())));
          LOG.debug("Done {} : {}", runnableName, url);
        }


        localFiles.put(runnableName,
                       new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(),
                                            location.length(), localFile.isArchive(), localFile.getPattern()));
      }
    }
    LOG.debug("Done Runnable LocalFiles");
  }

View Full Code Here

      }
    });


    // Serialize into a local temp file.
    LOG.debug("Create and copy {}", Constants.Files.TWILL_SPEC);
    Location location = createTempLocation(Constants.Files.TWILL_SPEC);
    Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
    try {
      EventHandlerSpecification eventHandler = spec.getEventHandler();
      if (eventHandler == null) {
        eventHandler = new LogOnlyEventHandler().configure();
      }

View Full Code Here

    localFiles.put(Constants.Files.TWILL_SPEC, createLocalFile(Constants.Files.TWILL_SPEC, location));
  }


  private void saveLogback(Map<String, LocalFile> localFiles) throws IOException {
    LOG.debug("Create and copy {}", Constants.Files.LOGBACK_TEMPLATE);
    Location location = copyFromURL(getClass().getClassLoader().getResource(Constants.Files.LOGBACK_TEMPLATE),
                                    createTempLocation(Constants.Files.LOGBACK_TEMPLATE));
    LOG.debug("Done {}", Constants.Files.LOGBACK_TEMPLATE);


    localFiles.put(Constants.Files.LOGBACK_TEMPLATE, createLocalFile(Constants.Files.LOGBACK_TEMPLATE, location));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.twill.filesystem.Location

co.cask.cdap.app.program.Programs

co.cask.cdap.data.runtime.LocationStreamFileWriterFactory

co.cask.cdap.data.stream.AbstractStreamCoordinator

co.cask.cdap.data.stream.LiveStreamFileReader

co.cask.cdap.data.stream.service.StreamFetchHandler

co.cask.cdap.data.stream.StreamFileJanitor

co.cask.cdap.data.stream.StreamTailer

co.cask.cdap.data.stream.StreamUtils

co.cask.cdap.data.stream.TimePartitionedStreamFileWriter$StreamWriterFactory

co.cask.cdap.data2.datafabric.dataset.RemoteDatasetFramework

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.