Package com.linkedin.camus.etl.kafka.mapred

Examples of com.linkedin.camus.etl.kafka.mapred.EtlSplit

47
48
49
50
51
52
53
54
55
56
57
    List<InputSplit> kafkaETLSplits = new ArrayList<InputSplit>();

    for (int i = 0; i < numTasks; i++) {
      if (requests.size() > 0) {
        kafkaETLSplits.add(new EtlSplit());
      }
    }

    for (CamusRequest r : requests) {
      getSmallestMultiSplit(kafkaETLSplits).addRequest(r);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    return kafkaETLSplits;
  }
 
  protected EtlSplit getSmallestMultiSplit(List<InputSplit> kafkaETLSplits)
      throws IOException {
    EtlSplit smallest = (EtlSplit) kafkaETLSplits.get(0);

    for (int i = 1; i < kafkaETLSplits.size(); i++) {
      EtlSplit challenger = (EtlSplit) kafkaETLSplits.get(i);
      if ((smallest.getLength() == challenger.getLength() && smallest
          .getNumRequests() > challenger.getNumRequests())
          || smallest.getLength() > challenger.getLength()) {
        smallest = challenger;
      }
    }

    return smallest;
View Full Code Here
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
        .getInt("mapred.map.tasks", 30);
    List<InputSplit> kafkaETLSplits = new ArrayList<InputSplit>();
   
    for (int i = 0; i < numTasks; i++) {
      if (requests.size() > 0) {
        kafkaETLSplits.add(new EtlSplit());
      }
    }
   
    List<CamusRequest> groupedRequests = groupSmallRequest(requests, context);
   
    reverseSortRequests(groupedRequests);

    for (CamusRequest r : groupedRequests) {
      EtlSplit split = getSmallestMultiSplit(kafkaETLSplits);
      for (CamusRequest r1 : (GroupedRequest)r) {
        split.addRequest(r1);
      }
    }

    return kafkaETLSplits;
  }
View Full Code Here
TOP

Related Classes of com.linkedin.camus.etl.kafka.mapred.EtlSplit

  • com.linkedin.camus.etl.kafka.common.EtlRequest
  • com.linkedin.camus.workallocater.BaseAllocator
  • com.linkedin.camus.workallocater.CamusRequest
  • com.linkedin.camus.workallocater.TopicGroupingAllocator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.