Package org.apache.sqoop.submission.counter

Examples of org.apache.sqoop.submission.counter.Counters

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    MSubmission target = transfer(source);
    assertEquals(25.0, target.getProgress());
  }

  public void testTransferCounters() {
    Counters counters = new Counters();
    counters.addCounterGroup(new CounterGroup("A")
      .addCounter(new Counter("X", 1))
      .addCounter(new Counter("Y", 2))
    );
    counters.addCounterGroup(new CounterGroup("B")
      .addCounter(new Counter("XX", 11))
      .addCounter(new Counter("YY", 22))
    );

    MSubmission source = new MSubmission();
    source.setCounters(counters);

    Counters target;
    CounterGroup group;
    Counter counter;

    target = transfer(source).getCounters();
    group = target.getCounterGroup("A");
    assertNotNull(group);
    counter = group.getCounter("X");
    assertNotNull(counter);
    assertEquals(1, counter.getValue());
    counter = group.getCounter("Y");
    assertNotNull(counter);
    assertEquals(2, counter.getValue());

    target = transfer(source).getCounters();
    group = target.getCounterGroup("B");
    assertNotNull(group);
    counter = group.getCounter("XX");
    assertNotNull(counter);
    assertEquals(11, counter.getValue());
    counter = group.getCounter("YY");
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
      } else {
        io.out.println(String.format("%.2f %%", progress));
      }
    }

    Counters counters = submission.getCounters();
    if(counters != null) {
      io.out.println("Counters:");
      for(CounterGroup group : counters) {
        io.out.print("\t");
        io.out.println(group.getName());
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    }
  }

  public Counters restoreCounters(JSONObject object) {
    Set<Map.Entry<String, JSONObject>> groupSet = object.entrySet();
    Counters counters = new Counters();

    for(Map.Entry<String, JSONObject> groupEntry: groupSet) {

      CounterGroup group = new CounterGroup(groupEntry.getKey());

      Set<Map.Entry<String, Long>> counterSet = groupEntry.getValue().entrySet();

      for(Map.Entry<String, Long> counterEntry: counterSet) {
        Counter counter = new Counter(counterEntry.getKey(), counterEntry.getValue());
        group.addCounter(counter);
      }

      counters.addCounterGroup(group);
    }

    return counters;
  }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    MSubmission target = transfer(source);
    assertEquals(25.0, target.getProgress());
  }

  public void testTransferCounters() {
    Counters counters = new Counters();
    counters.addCounterGroup(new CounterGroup("A")
      .addCounter(new Counter("X", 1))
      .addCounter(new Counter("Y", 2))
    );
    counters.addCounterGroup(new CounterGroup("B")
      .addCounter(new Counter("XX", 11))
      .addCounter(new Counter("YY", 22))
    );

    MSubmission source = new MSubmission();
    source.setCounters(counters);

    Counters target;
    CounterGroup group;
    Counter counter;

    target = transfer(source).getCounters();
    group = target.getCounterGroup("A");
    assertNotNull(group);
    counter = group.getCounter("X");
    assertNotNull(counter);
    assertEquals(1, counter.getValue());
    counter = group.getCounter("Y");
    assertNotNull(counter);
    assertEquals(2, counter.getValue());

    target = transfer(source).getCounters();
    group = target.getCounterGroup("B");
    assertNotNull(group);
    counter = group.getCounter("XX");
    assertNotNull(counter);
    assertEquals(11, counter.getValue());
    counter = group.getCounter("YY");
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    CounterGroup secondGroup = new CounterGroup("gb");
    firstGroup.addCounter(new Counter("ca", 100));
    firstGroup.addCounter(new Counter("cb", 200));
    secondGroup.addCounter(new Counter("ca", 300));
    secondGroup.addCounter(new Counter("cd", 400));
    Counters counters = new Counters();
    counters.addCounterGroup(firstGroup);
    counters.addCounterGroup(secondGroup);

    MSubmission submission = new MSubmission();
    submission.setJobId(1);
    submission.setStatus(SubmissionStatus.RUNNING);
    submission.setCreationDate(creationDate);
    submission.setLastUpdateDate(updateDate);
    submission.setExternalId("job-x");
    submission.setExternalLink("http://somewhere");
    submission.setExceptionInfo("RuntimeException");
    submission.setExceptionStackTrace("Yeah it happens");
    submission.setCounters(counters);

    handler.createSubmission(submission, getDerbyConnection());

    assertEquals(1, submission.getPersistenceId());
    assertCountForTable("SQOOP.SQ_SUBMISSION", 1);

    List<MSubmission> submissions =
      handler.findSubmissionsUnfinished(getDerbyConnection());
    assertNotNull(submissions);
    assertEquals(1, submissions.size());

    submission = submissions.get(0);

    assertEquals(1, submission.getJobId());
    assertEquals(SubmissionStatus.RUNNING, submission.getStatus());
    assertEquals(creationDate, submission.getCreationDate());
    assertEquals(updateDate, submission.getLastUpdateDate());
    assertEquals("job-x", submission.getExternalId());
    assertEquals("http://somewhere", submission.getExternalLink());
    assertEquals("RuntimeException", submission.getExceptionInfo());
    assertEquals("Yeah it happens", submission.getExceptionStackTrace());

    CounterGroup group;
    Counter counter;
    Counters retrievedCounters = submission.getCounters();
    assertNotNull(retrievedCounters);

    group = counters.getCounterGroup("ga");
    assertNotNull(group);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    }
  }

  public Counters restoreCounters(JSONObject object) {
    Set<Map.Entry<String, JSONObject>> groupSet = object.entrySet();
    Counters counters = new Counters();

    for(Map.Entry<String, JSONObject> groupEntry: groupSet) {

      CounterGroup group = new CounterGroup(groupEntry.getKey());

      Set<Map.Entry<String, Long>> counterSet = groupEntry.getValue().entrySet();

      for(Map.Entry<String, Long> counterEntry: counterSet) {
        Counter counter = new Counter(counterEntry.getKey(), counterEntry.getValue());
        group.addCounter(counter);
      }

      counters.addCounterGroup(group);
    }

    return counters;
  }
View Full Code Here
562
563
564
565
566
567
568
569
570
571
572
    return submission;
  }

  private void update(MSubmission submission) {
    double progress  = -1;
    Counters counters = null;
    String externalId = submission.getExternalId();
    SubmissionStatus newStatus = submissionEngine.status(externalId);
    String externalLink = submissionEngine.externalLink(externalId);

    if(newStatus.isRunning()) {
View Full Code Here
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
   *
   * @param hadoopCounters Hadoop counters
   * @return Appropriate Sqoop counters
   */
  private Counters convertMapreduceCounters(org.apache.hadoop.mapred.Counters hadoopCounters) {
    Counters sqoopCounters = new Counters();

    for(org.apache.hadoop.mapred.Counters.Group hadoopGroup : hadoopCounters) {
      CounterGroup sqoopGroup = new CounterGroup(hadoopGroup.getName());
      for(org.apache.hadoop.mapred.Counters.Counter hadoopCounter : hadoopGroup) {
        Counter sqoopCounter = new Counter(hadoopCounter.getName(), hadoopCounter.getValue());
        sqoopGroup.addCounter(sqoopCounter);
      }
      sqoopCounters.addCounterGroup(sqoopGroup);
    }

    return sqoopCounters;
  }
View Full Code Here
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
    submission.setExternalId(rs.getString(6));
    submission.setExternalLink(rs.getString(7));
    submission.setExceptionInfo(rs.getString(8));
    submission.setExceptionStackTrace(rs.getString(9));

    Counters counters = loadCountersSubmission(rs.getLong(1), conn);
    submission.setCounters(counters);

    return submission;
  }
View Full Code Here
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
    try {
      stmt = conn.prepareStatement(STMT_SELECT_COUNTER_SUBMISSION);
      stmt.setLong(1, submissionId);
      rs = stmt.executeQuery();

      Counters counters = new Counters();

      while(rs.next()) {
        String groupName = rs.getString(1);
        String counterName = rs.getString(2);
        long value = rs.getLong(3);

        CounterGroup group = counters.getCounterGroup(groupName);
        if(group == null) {
          group = new CounterGroup(groupName);
          counters.addCounterGroup(group);
        }

        group.addCounter(new Counter(counterName, value));
      }

      if(counters.isEmpty()) {
        return null;
      } else {
        return counters;
      }
    } finally {
View Full Code Here
TOP

Related Classes of org.apache.sqoop.submission.counter.Counters

  • org.apache.sqoop.client.utils.SubmissionDisplayer
  • org.apache.sqoop.framework.FrameworkManager
  • org.apache.sqoop.framework.JobManager
  • org.apache.sqoop.json.SubmissionBean
  • org.apache.sqoop.json.TestSubmissionBean
  • org.apache.sqoop.repository.derby.DerbyRepositoryHandler
  • org.apache.sqoop.repository.derby.TestSubmissionHandling
  • org.apache.sqoop.shell.utils.SubmissionDisplayer
  • org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.