Package co.cask.cdap.data2.transaction.queue.hbase.coprocessor

Examples of co.cask.cdap.data2.transaction.queue.hbase.coprocessor.QueueConsumerConfig

68
69
70
71
72
73
74
75
76
  public QueueConsumer createConsumer(QueueName queueName, ConsumerConfig consumerConfig, int numGroups)
    throws IOException {
    LevelDBQueueAdmin admin = ensureTableExists(queueName);
    LevelDBOrderedTableCore core = new LevelDBOrderedTableCore(admin.getActualTableName(queueName), service);
    // only the first consumer of each group runs eviction; and only if the number of consumers is known (> 0).
    QueueEvictor evictor = (numGroups <= 0 || consumerConfig.getInstanceId() != 0) ? QueueEvictor.NOOP :
      new LevelDBQueueEvictor(core, queueName, numGroups, evictionExecutor);
    return new LevelDBQueueConsumer(core, getQueueLock(queueName.toString()), consumerConfig, queueName, evictor);
  }
View Full Code Here
456
457
458
459
460
461
462
463
464
465
466
            if (queueSpec.getQueueName().getSimpleName().equals(outputName)
                && queueSpec.getOutputSchema().equals(schema)) {

              final String queueMetricsName = "process.events.out";
              final String queueMetricsTag = queueSpec.getQueueName().getSimpleName();
              QueueProducer producer = queueClientFactory.createProducer(queueSpec.getQueueName(), new QueueMetrics() {
                @Override
                public void emitEnqueue(int count) {
                  flowletContext.getProgramMetrics().increment(queueMetricsName, count, queueMetricsTag);
                }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
  }

  @Override
  protected QueueScanner getScanner(byte[] startRow, byte[] stopRow, int numRows) throws IOException {
    final Scanner scanner = core.scan(startRow, stopRow, null, null, Transaction.ALL_VISIBLE_LATEST);
    return new QueueScanner() {
      @Override
      public ImmutablePair<byte[], Map<byte[], byte[]>> next() throws IOException {
        Row next = scanner.next();
        if (next == null) {
          return null;
View Full Code Here
116
117
118
119
120
121
122
    private EvictionInternalScanner(String action, RegionCoprocessorEnvironment env, InternalScanner scanner) {
      this.triggeringAction = action;
      this.env = env;
      this.scanner = scanner;
      this.consumerInstance = new ConsumerInstance(0, 0);
    }
View Full Code Here
119
120
121
122
123
124
125
    private EvictionInternalScanner(String action, RegionCoprocessorEnvironment env, InternalScanner scanner) {
      this.triggeringAction = action;
      this.env = env;
      this.scanner = scanner;
      this.consumerInstance = new ConsumerInstance(0, 0);
    }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
  public StreamConsumer createStreamConsumer(QueueName streamName, ConsumerConfig consumerConfig) throws IOException {
    String namespace = String.format("%s.%s", programId.getApplicationId(), programId.getId());
    final StreamConsumer consumer = streamConsumerFactory.create(streamName, namespace, consumerConfig);

    dataSetContext.addTransactionAware(consumer);
    return new ForwardingStreamConsumer(consumer) {
      @Override
      public void close() throws IOException {
        super.close();
        dataSetContext.removeTransactionAware(consumer);
      }
View Full Code Here
44
45
46
47
48
49
50
  @Override
  public StreamConsumer create(QueueName streamName, String namespace,
                               ConsumerConfig consumerConfig) throws IOException {

    QueueConsumer consumer = queueClientFactory.createConsumer(streamName, consumerConfig, -1);
    return new QueueToStreamConsumer(streamName, consumerConfig, consumer);
  }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf),
                                             new DataFabricModules().getDistributedModules(),
                                             new DataSetsModules().getDistributedModule(),
                                             new LocationRuntimeModule().getDistributedModules());

    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    StreamConfig streamConfig = streamAdmin.getConfig(streamName);
    Location streamLocation = streamConfig.getLocation();
    List<Location> eventFiles = Lists.newArrayList();

    for (Location partition : streamLocation.list()) {
      if (!partition.isDirectory()) {
View Full Code Here
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
    if (inputDataSetName != null) {
      // TODO: It's a hack for stream
      if (inputDataSetName.startsWith("stream://")) {
        StreamBatchReadable stream = new StreamBatchReadable(URI.create(inputDataSetName));
        StreamConfig streamConfig = streamAdmin.getConfig(stream.getStreamName());
        Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(),
                                                                   StreamUtils.getGeneration(streamConfig));

        LOG.info("Using Stream as input from {}", streamPath.toURI());

        TextStreamInputFormat.setTTL(job, streamConfig.getTTL());
        TextStreamInputFormat.setStreamPath(job, streamPath.toURI());
        TextStreamInputFormat.setTimeRange(job, stream.getStartTime(), stream.getEndTime());
        job.setInputFormatClass(TextStreamInputFormat.class);

      } else {
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
            try {
              int currentGeneration = (property == null) ?
                StreamUtils.getGeneration(streamConfig) :
                property.getGeneration();

              StreamConfig newConfig = new StreamConfig(streamConfig.getName(), streamConfig.getPartitionDuration(),
                                                        streamConfig.getIndexInterval(), newTTL,
                                                        streamConfig.getLocation());
              saveConfig(newConfig);
              resultFuture.set(new StreamProperty(currentGeneration, newTTL));
            } catch (IOException e) {
View Full Code Here
TOP

Related Classes of co.cask.cdap.data2.transaction.queue.hbase.coprocessor.QueueConsumerConfig

  • co.cask.cdap.api.dataset.DatasetAdmin
  • co.cask.cdap.api.dataset.DatasetDefinition
  • co.cask.cdap.api.dataset.DatasetSpecification
  • co.cask.cdap.api.dataset.table.ConflictDetection
  • co.cask.cdap.api.dataset.table.Result
  • co.cask.cdap.api.dataset.table.Row
  • co.cask.cdap.api.dataset.table.Scanner
  • co.cask.cdap.api.security.ACL
  • co.cask.cdap.common.hooks.MetricsReporterHook
  • co.cask.cdap.common.http.HttpRequest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.