Package com.linkedin.camus.etl.kafka.common

Examples of com.linkedin.camus.etl.kafka.common.EtlKey

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
        addOffset(key);
    }

    public void addOffset(EtlKey key) {
        String topicPart = key.getTopic() + "-" + key.getLeaderId() + "-" + key.getPartition();
        EtlKey offsetKey = new EtlKey(key);
       
        if (offsets.containsKey(topicPart)){
          long avgSize = offsets.get(topicPart).getMessageSize() * eventCounts.get(topicPart) + key.getMessageSize();
          avgSize /= eventCounts.get(topicPart) + 1;
          offsetKey.setMessageSize(avgSize);
        } else {
          eventCounts.put(topicPart, 0l);
        }
        eventCounts.put(topicPart, eventCounts.get(topicPart) + 1);
        offsets.put(topicPart, offsetKey);
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
    for (CamusRequest request : finalRequests) {
      if (moveLatest.contains(request.getTopic())
          || moveLatest.contains("all")) {
          log.info("Moving to latest for topic: " + request.getTopic());
        //TODO: factor out kafka specific request functionality
          EtlKey oldKey = offsetKeys.get(request);
          EtlKey newKey = new EtlKey(request.getTopic(), ((EtlRequest)request).getLeaderId(),
              request.getPartition(), 0, request
              .getLastOffset());
         
          if (oldKey != null)
            newKey.setMessageSize(oldKey.getMessageSize());
         
          offsetKeys.put(request, newKey);
      }

      EtlKey key = offsetKeys.get(request);

      if (key != null) {
        request.setOffset(key.getOffset());
        request.setAvgMsgSize(key.getMessageSize());
      }

      if (request.getEarliestOffset() > request.getOffset()
          || request.getOffset() > request.getLastOffset()) {
        if(request.getEarliestOffset() > request.getOffset())
        {
          log.error("The earliest offset was found to be more than the current offset: " + request);
          log.error("Moving to the earliest offset available");
        }
        else
        {
          log.error("The current offset was found to be more than the latest offset: " + request);
          log.error("Moving to the earliest offset available");
        }
        request.setOffset(request.getEarliestOffset());
        offsetKeys.put(
            request,
          //TODO: factor out kafka specific request functionality
            new EtlKey(request.getTopic(), ((EtlRequest)request).getLeaderId(),
                request.getPartition(), 0, request
                    .getOffset()));
      }
      log.info(request);
    }
View Full Code Here
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
      FileSystem fs = input.getFileSystem(context.getConfiguration());
      for (FileStatus f : fs.listStatus(input, new OffsetFileFilter())) {
        log.info("previous offset file:" + f.getPath().toString());
        SequenceFile.Reader reader = new SequenceFile.Reader(fs,
            f.getPath(), context.getConfiguration());
        EtlKey key = new EtlKey();
        while (reader.next(key, NullWritable.get())) {
        //TODO: factor out kafka specific request functionality
          CamusRequest request = new EtlRequest(context,
              key.getTopic(), key.getLeaderId(),
              key.getPartition());
          if (offsetKeysMap.containsKey(request)) {

            EtlKey oldKey = offsetKeysMap.get(request);
            if (oldKey.getOffset() < key.getOffset()) {
              offsetKeysMap.put(request, key);
            }
          } else {
            offsetKeysMap.put(request, key);
          }
          key = new EtlKey();
        }
        reader.close();
      }
    }
    return offsetKeysMap;
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
        assertTrue(actualResult.equals(expectedResult));
    }

    @Test
    public void testEncodedPartition() throws IOException {
        EtlKey testEtlKey = new EtlKey();
        testEtlKey.setTime(1400549463000L);
        Configuration testConfiguration = new Configuration();
        Job testJob = new Job(new Configuration());

        DefaultPartitioner testPartitioner = new DefaultPartitioner();
        testPartitioner.setConf(testConfiguration);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
      SequenceFile.Reader reader = new SequenceFile.Reader(fs,
          f.getPath(), fs.getConf());

            String errorFrom = "\nError from file [" + f.getPath() + "]";

      EtlKey key = new EtlKey();
      ExceptionWritable value = new ExceptionWritable();

      while (reader.next(key, value)) {
                ExceptionWritable exceptionWritable = new ExceptionWritable(value.toString() + errorFrom);
                errors.put(new EtlKey(key), exceptionWritable);
      }
      reader.close();
    }

        return errors;
View Full Code Here
TOP

Related Classes of com.linkedin.camus.etl.kafka.common.EtlKey

  • com.linkedin.camus.etl.kafka.CamusJob
  • com.linkedin.camus.etl.kafka.coders.TestDefaultPartitioner
  • com.linkedin.camus.etl.kafka.mapred.EtlInputFormat
  • com.linkedin.camus.etl.kafka.mapred.EtlMultiOutputCommitter
  • org.apache.hadoop.io.LongWritable
  • org.apache.hadoop.io.MapWritable
  • org.apache.hadoop.io.Text
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.