Package org.apache.twill.filesystem

Examples of org.apache.twill.filesystem.LocationFactory


      dataIngestionPortsMap.put(Constants.TCP_INGESTION_PORT_PREFIX + inputName, tcpPort);
    }

    // Setup temporary directory structure
    tmpFolder = Files.createTempDir();
    LocationFactory locationFactory = new LocalLocationFactory(tmpFolder);

    Location baseDir = locationFactory.create("baseDir");
    baseDir.mkdirs();

    InputFlowletConfiguration inputFlowletConfiguration = new LocalInputFlowletConfiguration(baseDir, spec);
    Location binDir = inputFlowletConfiguration.createStreamEngineProcesses();
View Full Code Here


   * that load classes from all the ProgramClassLoader and the URLClassLoader as described above.
   */
  public ClassLoader makeClassLoader(final ClassLoader parent)
    throws MalformedURLException {

    final LocationFactory lf = new LocalLocationFactory();
    final List<ClassLoader> classLoaders = Lists.newArrayList();
    final List<URL> urls = Lists.newArrayList();
    for (final String classPath : localClasspaths) {
      final URI uri = new File(classPath).toURI();
      if (classPath.endsWith(".jar")) {
        classLoaders.add(AccessController.doPrivileged(new PrivilegedAction<ClassLoader>() {
          @Override
          public ClassLoader run() {
            try {
              File expandDir = Files.createTempDir();
              jarExpandDirs.add(expandDir);
              return ClassLoaders.newProgramClassLoader(
                BundleJarUtil.unpackProgramJar(lf.create(uri), expandDir),
                ApiResourceListHolder.getResourceList(), parent);
            } catch (IOException e) {
              throw Throwables.propagate(e);
            }
          }
View Full Code Here

  public int doMain(String[] args) {
    String jarFilename;
    File outputFile;
    String locationFactory;
    String id;
    LocationFactory lf;

    CommandLineParser parser = new GnuParser();
    Options options = new Options();
    options.addOption("i", "id", true, "Account ID");
    options.addOption("j", "jar", true, "Application JAR");
    options.addOption("f", "locfactory", true, "Location Factory (LOCAL or DISTRIBUTED)");
    options.addOption("o", "output", true, "Output");

    // Check all the options of command line
    try {
      CommandLine line = parser.parse(options, args);
      if (!line.hasOption("jar")) {
        LOG.error("Application JAR not specified.");
        return -1;
      }
      if (!line.hasOption("output")) {
        LOG.error("Output file not specified.");
        return -1;
      }
      if (!line.hasOption("id")) {
        LOG.error("Account id not specified.");
        return -1;
      }
      if (!line.hasOption("locfactory")) {
        LOG.error("Location factory not specified.");
        return -1;
      }

      jarFilename = line.getOptionValue("jar");
      outputFile = new File(line.getOptionValue("output"));
      id = line.getOptionValue("id");
      locationFactory = line.getOptionValue("locfactory");
    } catch (ParseException e) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp("SandboxJVM", options);
      return -1;
    }

    // Load the JAR using the JAR class load and load the manifest file.
    File unpackedJarDir = Files.createTempDir();

    try {
      Application app;
      try {
        if ("LOCAL".equals(locationFactory)) {
          lf = new LocalLocationFactory();
        } else if ("DISTRIBUTED".equals(locationFactory)) {
          lf = new HDFSLocationFactory(new Configuration());
        } else {
          LOG.error("Unknown location factory specified");
          return -1;
        }

        Program archive = Programs.createWithUnpack(lf.create(jarFilename), unpackedJarDir);
        Object appMain = archive.getMainClass().newInstance();
        if (!(appMain instanceof Application)) {
          LOG.error(String.format("Application main class is of invalid type: %s",
                                  appMain.getClass().getName()));
          return -1;
View Full Code Here

   *
   * @param eventLocation Location for the event file.
   * @return Location of the index file.
   */
  private Location createIndexLocation(Location eventLocation) {
    LocationFactory factory = eventLocation.getLocationFactory();
    String eventPath = eventLocation.toURI().toString();
    int extLength = StreamFileType.EVENT.getSuffix().length();

    return factory.create(URI.create(String.format("%s%s",
                                                   eventPath.substring(0, eventPath.length() - extLength),
                                                   StreamFileType.INDEX.getSuffix())));
  }
View Full Code Here

    metricsQueryService.startAndWait();
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    AppFabricHttpHandler httpHandler = injector.getInstance(AppFabricHttpHandler.class);
    ServiceHttpHandler serviceHttpHandler = injector.getInstance(ServiceHttpHandler.class);
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    appFabricClient = new AppFabricClient(httpHandler, serviceHttpHandler, locationFactory);
    DatasetFramework dsFramework = injector.getInstance(DatasetFramework.class);
    datasetFramework =
      new NamespacedDatasetFramework(dsFramework,
                                     new DefaultDatasetNamespace(cConf,  Namespace.USER));
View Full Code Here

                                     @Nullable List<Split> inputSplits,
                                     @Nullable String outputDataSetName) {
    Injector injector = prepare();

    // Initializing Program
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    Program program;
    try {
      program = Programs.create(locationFactory.create(programLocation), classLoader);
      // See if it is launched from Workflow, if it is, change the Program.
      if (workflowBatch != null) {
        MapReduceSpecification mapReduceSpec = program.getSpecification().getMapReduce().get(workflowBatch);
        Preconditions.checkArgument(mapReduceSpec != null, "Cannot find MapReduceSpecification for %s", workflowBatch);
        program = new WorkflowMapReduceProgram(program, mapReduceSpec);
View Full Code Here

  public BasicSparkContext build(String runId, long logicalStartTime, String workflowBatch, Arguments runtimeArguments,
                                 Transaction tx, ClassLoader classLoader, URI programLocation) {
    Injector injector = prepare();

    // Initializing Program
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    Program program;
    try {
      program = Programs.create(locationFactory.create(programLocation), classLoader);
      //TODO: This should be changed when we support Spark in Workflow
    } catch (IOException e) {
      LOG.error("Could not init Program based on location: " + programLocation);
      throw Throwables.propagate(e);
    }
View Full Code Here

TOP

Related Classes of org.apache.twill.filesystem.LocationFactory

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.