Examples of org.apache.twill.filesystem.Location

org.apache.twill.filesystem.Location
This interface defines the location and operations of a resource on the filesystem.
{@link Location} is agnostic to the type of file system the resource is on.

                         TxConstants.DataJanitor.DEFAULT_TX_JANITOR_ENABLE)) {
      return CoprocessorJar.EMPTY;
    }


    // create the jar for the data janitor coprocessor.
    Location jarDir = locationFactory.create(conf.get(Constants.CFG_HDFS_LIB_DIR));
    Class<? extends Coprocessor> dataJanitorClass = tableUtil.getTransactionDataJanitorClassForVersion();
    Class<? extends Coprocessor> incrementClass = tableUtil.getIncrementHandlerClassForVersion();
    ImmutableList.Builder<Class<? extends Coprocessor>> coprocessors = ImmutableList.builder();
    coprocessors.add(dataJanitorClass);
    if (supportsReadlessIncrement) {
      coprocessors.add(incrementClass);
    }
    ImmutableList<Class<? extends Coprocessor>> coprocessorList = coprocessors.build();
    Location jarFile = HBaseTableUtil.createCoProcessorJar("table", jarDir, coprocessorList);
    return new CoprocessorJar(coprocessorList, jarFile);
  }

View Full Code Here

   *
   * @return the partition location if found, or {@code null} if not found.
   */
  private Location getStartPartitionLocation(StreamConfig streamConfig,
                                             long startTime, int generation) throws IOException {
    Location baseLocation = StreamUtils.createGenerationLocation(streamConfig.getLocation(), generation);


    // Find the partition location with the largest timestamp that is <= startTime
    // Also find the partition location with the smallest timestamp that is >= startTime
    long maxPartitionStartTime = 0L;
    long minPartitionStartTime = Long.MAX_VALUE;


    Location maxPartitionLocation = null;
    Location minPartitionLocation = null;


    for (Location location : baseLocation.list()) {
      // Partition must be a directory
      if (!location.isDirectory()) {
        continue;

View Full Code Here

   * Creates a {@link FileReader} that starts reading stream event from the given partition.
   */
  private FileReader<StreamEventOffset, Iterable<StreamFileOffset>> createReader(StreamConfig streamConfig,
                                                                                 long startTime) throws IOException {
    int generation = StreamUtils.getGeneration(streamConfig);
    Location startPartition = getStartPartitionLocation(streamConfig, startTime, generation);
    if (startPartition == null) {
      return createEmptyReader();
    }


    List<StreamFileOffset> fileOffsets = Lists.newArrayList();
    int instances = cConf.getInt(Constants.Stream.CONTAINER_INSTANCES);
    String filePrefix = cConf.get(Constants.Stream.FILE_PREFIX);
    for (int i = 0; i < instances; i++) {
      // The actual file prefix is formed by file prefix in cConf + writer instance id
      String streamFilePrefix = filePrefix + '.' + i;
      Location eventLocation = StreamUtils.createStreamLocation(startPartition, streamFilePrefix,
                                                                0, StreamFileType.EVENT);
      fileOffsets.add(new StreamFileOffset(eventLocation, 0, generation));
    }


    MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, fileOffsets);

View Full Code Here

      LOG.warn("Cannot add module {}: content is null", name);
      responder.sendString(HttpResponseStatus.BAD_REQUEST, "Content is null");
      return;
    }


    Location uploadDir = locationFactory.create(archiveDir).append("account_placeholder");
    String archiveName = name + ".jar";
    Location archive = uploadDir.append(archiveName);
    LOG.info("Storing module {} jar at {}", name, archive.toURI().toString());


    if (!uploadDir.exists() && !uploadDir.mkdirs()) {
      LOG.warn("Unable to create directory '{}'", uploadDir.getName());
    }


    InputStream inputStream = new ChannelBufferInputStream(content);
    try {
      // todo: store to temp file first and do some verifications? Or even datasetFramework should persist file?
      OutputStream outStream = archive.getOutputStream();
      try {
        ByteStreams.copy(inputStream, outStream);
      } finally {
        outStream.close();
      }

View Full Code Here

      return;
    }


    // Generate the coprocessor jar
    CoprocessorJar coprocessorJar = createCoprocessorJar();
    Location jarLocation = coprocessorJar.getJarLocation();


    // Check if coprocessor upgrade is needed
    Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);


    // For all required coprocessors, check if they've need to be upgraded.
    for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
      HBaseTableUtil.CoprocessorInfo info = coprocessorInfo.get(coprocessor.getName());
      if (info != null) {
        // The same coprocessor has been configured, check by the file name hash to see if they are the same.
        if (!jarLocation.getName().equals(info.getPath().getName())) {
          needUpgrade = true;
          // Remove old one and add the new one.
          tableDescriptor.removeCoprocessor(info.getClassName());
          addCoprocessor(tableDescriptor, coprocessor, jarLocation, coprocessorJar.getPriority(coprocessor));
        }

View Full Code Here

    oldStreamAdmin.upgrade();
  }


  @Override
  public StreamConfig getConfig(String streamName) throws IOException {
    Location streamLocation = streamBaseLocation.append(streamName);
    Preconditions.checkArgument(streamLocation.isDirectory(), "Stream '%s' not exists.", streamName);
    return loadConfig(streamLocation);
  }

View Full Code Here

    return loadConfig(streamLocation);
  }


  @Override
  public void updateConfig(StreamConfig config) throws IOException {
    Location streamLocation = config.getLocation();
    Preconditions.checkArgument(streamLocation.isDirectory(), "Stream '{}' not exists.", config.getName());


    // Check only TTL is changed, as only TTL change is supported.
    StreamConfig originalConfig = loadConfig(streamLocation);
    Preconditions.checkArgument(isValidConfigUpdate(originalConfig, config),
                                "Configuration update for stream '{}' was not valid (can only update ttl)",

View Full Code Here

    create(name, null);
  }


  @Override
  public void create(String name, @Nullable Properties props) throws Exception {
    Location streamLocation = streamBaseLocation.append(name);
    Locations.mkdirsIfNotExists(streamLocation);


    Location configLocation = streamBaseLocation.append(name).append(CONFIG_FILE_NAME);
    if (!configLocation.createNew()) {
      // Stream already exists
      return;
    }


    Properties properties = (props == null) ? new Properties() : props;
    long partitionDuration = Long.parseLong(properties.getProperty(Constants.Stream.PARTITION_DURATION,
                                            cConf.get(Constants.Stream.PARTITION_DURATION)));
    long indexInterval = Long.parseLong(properties.getProperty(Constants.Stream.INDEX_INTERVAL,
                                                               cConf.get(Constants.Stream.INDEX_INTERVAL)));
    long ttl = Long.parseLong(properties.getProperty(Constants.Stream.TTL,
                                                     cConf.get(Constants.Stream.TTL)));


    Location tmpConfigLocation = configLocation.getTempFile(null);
    StreamConfig config = new StreamConfig(name, partitionDuration, indexInterval, ttl, streamLocation);
    CharStreams.write(GSON.toJson(config), CharStreams.newWriterSupplier(
      Locations.newOutputSupplier(tmpConfigLocation), Charsets.UTF_8));


    try {
      // Windows does not allow renaming if the destination file exists so we must delete the configLocation
      if (OSDetector.isWindows()) {
        configLocation.delete();
      }
      tmpConfigLocation.renameTo(streamBaseLocation.append(name).append(CONFIG_FILE_NAME));
    } finally {
      if (tmpConfigLocation.exists()) {
        tmpConfigLocation.delete();
      }
    }
  }

View Full Code Here

      oldStreamAdmin.upgrade(streamName, properties);
    }
  }


  private StreamConfig loadConfig(Location streamLocation) throws IOException {
    Location configLocation = streamLocation.append(CONFIG_FILE_NAME);


    StreamConfig config = GSON.fromJson(
      CharStreams.toString(CharStreams.newReaderSupplier(Locations.newInputSupplier(configLocation), Charsets.UTF_8)),
      StreamConfig.class);

View Full Code Here

    int instances = cConf.getInt(Constants.Stream.CONTAINER_INSTANCES);
    String filePrefix = cConf.get(Constants.Stream.FILE_PREFIX);
    for (int i = 0; i < instances; i++) {
      // The actual file prefix is formed by file prefix in cConf + writer instance id
      String streamFilePrefix = filePrefix + '.' + i;
      Location eventLocation = StreamUtils.createStreamLocation(partitionLocation, streamFilePrefix,
                                                                0, StreamFileType.EVENT);
      fileOffsets.add(new StreamFileOffset(eventLocation, 0, generation));
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.twill.filesystem.Location

co.cask.cdap.app.program.Programs

co.cask.cdap.data.runtime.LocationStreamFileWriterFactory

co.cask.cdap.data.stream.AbstractStreamCoordinator

co.cask.cdap.data.stream.LiveStreamFileReader

co.cask.cdap.data.stream.service.StreamFetchHandler

co.cask.cdap.data.stream.StreamFileJanitor

co.cask.cdap.data.stream.StreamTailer

co.cask.cdap.data.stream.StreamUtils

co.cask.cdap.data.stream.TimePartitionedStreamFileWriter$StreamWriterFactory

co.cask.cdap.data2.datafabric.dataset.RemoteDatasetFramework

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.