Package co.cask.cdap.data2.datafabric.dataset.service.executor

Examples of co.cask.cdap.data2.datafabric.dataset.service.executor.RemoteDatasetOpExecutor

245
246
247
248
249
250
251
252
253
254
255
256
    private StreamProperty currentProperty;

    private StreamPropertyChangeListener(StreamAdmin streamAdmin, String streamName, StreamPropertyListener listener) {
      this.listener = listener;
      try {
        StreamConfig streamConfig = streamAdmin.getConfig(streamName);
        this.currentProperty = new StreamProperty(StreamUtils.getGeneration(streamConfig), streamConfig.getTTL());
      } catch (Exception e) {
        // It's ok if the stream config is not yet available (meaning no data has ever been writen to the stream yet.
        this.currentProperty = new StreamProperty(0, Long.MAX_VALUE);
      }
    }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
144
  public void getInfo(HttpRequest request, HttpResponder responder,
                      @PathParam("stream") String stream) throws Exception {
    String accountID = getAuthenticatedAccountId(request);

    if (streamMetaStore.streamExists(accountID, stream)) {
      StreamConfig streamConfig = streamAdmin.getConfig(stream);
      StreamProperties streamProperties = new StreamProperties(streamConfig.getName(), streamConfig.getTTL());
      responder.sendJson(HttpResponseStatus.OK, streamProperties, StreamProperties.class, GSON);
    } else {
      responder.sendStatus(HttpResponseStatus.NOT_FOUND);
    }
  }
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
236
237
238
      responder.sendString(HttpResponseStatus.NOT_FOUND, "Stream does not exists");
      return;
    }

    try {
      StreamConfig config = streamAdmin.getConfig(stream);
      try {
        config = getConfigUpdate(request, config);
        if (config.getTTL() < 0) {
          responder.sendString(HttpResponseStatus.BAD_REQUEST, "TTL value should be positive");
          return;
        }
      } catch (Throwable t) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, "Invalid stream configuration");
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
    // Only pickup changes in TTL
    if (json.has("ttl")) {
      JsonElement ttl = json.get("ttl");
      if (ttl.isJsonPrimitive()) {
        // TTL in the REST API is in seconds. Convert it to ms for the config.
        return new StreamConfig(config.getName(), config.getPartitionDuration(), config.getIndexInterval(),
                                TimeUnit.SECONDS.toMillis(ttl.getAsLong()), config.getLocation());
      }
    }
    return config;
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
                                             new DataFabricModules().getDistributedModules(),
                                             new DataSetsModules().getDistributedModule(),
                                             new LocationRuntimeModule().getDistributedModules());

    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    StreamConfig streamConfig = streamAdmin.getConfig(streamName);
    Location streamLocation = streamConfig.getLocation();
    List<Location> eventFiles = Lists.newArrayList();

    for (Location partition : streamLocation.list()) {
      if (!partition.isDirectory()) {
        continue;
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
    Supplier<FileWriter<StreamEvent>> create(final String streamName) {
      return new Supplier<FileWriter<StreamEvent>>() {
        @Override
        public FileWriter<StreamEvent> get() {
          try {
            StreamConfig streamConfig = streamAdmin.getConfig(streamName);
            Integer generation = generations.get(streamName);
            if (generation == null) {
              generation = StreamUtils.getGeneration(streamConfig);
            }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
    if (!verifyGetEventsRequest(accountID, stream, startTime, endTime, limit, responder)) {
      return;
    }

    StreamConfig streamConfig = streamAdmin.getConfig(stream);
    startTime = Math.max(startTime, System.currentTimeMillis() - streamConfig.getTTL());

    // Create the stream event reader
    FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader = createReader(streamConfig, startTime);
    try {
      ReadFilter readFilter = createReadFilter(startTime, endTime);
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
  }

  @Override
  public StreamConsumer createStreamConsumer(QueueName streamName, ConsumerConfig consumerConfig) throws IOException {
    String namespace = String.format("%s.%s", programId.getApplicationId(), programId.getId());
    final StreamConsumer consumer = streamConsumerFactory.create(streamName, namespace, consumerConfig);

    dataSetContext.addTransactionAware(consumer);
    return new ForwardingStreamConsumer(consumer) {
      @Override
      public void close() throws IOException {
View Full Code Here
43
44
45
46
47
48
49
50
51
    this.eventTransform = eventTransform;
  }

  @Override
  public InputDatum<T> dequeue(long timeout, TimeUnit timeoutUnit) throws IOException, InterruptedException {
    StreamConsumer consumer = consumerSupplier.get();
    return new BasicInputDatum<StreamEvent, T>(consumer.getStreamName(),
                                               consumer.poll(batchSize, timeout, timeoutUnit), eventTransform);
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
      if (queueName.isQueue()) {
        QueueConsumer queueConsumer = dataFabricFacade.createConsumer(queueName, config, numGroups);
        consumerConfig = queueConsumer.getConfig();
        consumer = queueConsumer;
      } else {
        StreamConsumer streamConsumer = dataFabricFacade.createStreamConsumer(queueName, config);
        consumerConfig = streamConsumer.getConsumerConfig();
        consumer = streamConsumer;
      }
    } catch (Exception e) {
      throw Throwables.propagate(e);
    }
View Full Code Here
TOP

Related Classes of co.cask.cdap.data2.datafabric.dataset.service.executor.RemoteDatasetOpExecutor

  • co.cask.cdap.api.dataset.DatasetAdmin
  • co.cask.cdap.api.dataset.DatasetDefinition
  • co.cask.cdap.api.dataset.DatasetSpecification
  • co.cask.cdap.api.dataset.module.DatasetModule
  • co.cask.cdap.api.dataset.table.Result
  • co.cask.cdap.api.dataset.table.Row
  • co.cask.cdap.api.dataset.table.Scanner
  • co.cask.cdap.api.security.ACL
  • co.cask.cdap.common.hooks.MetricsReporterHook
  • co.cask.cdap.common.http.HttpRequest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.