Package org.apache.hadoop.chukwa.extraction.engine

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

64
65
66
67
68
69
70
71
72
73
74
    String cus_reduceType = "someReduceType";
    conf.set(cus_reduceType, ",org.apache.hadoop.chukwa.extraction.demux.processor.reducer" +
            ".MockReduceProcessor");
    reducer.configure(conf);

    ChukwaRecordKey key = new ChukwaRecordKey(cus_reduceType, "someKey");
    ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output =
            new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();

    reducer.reduce(key, null, output, Reporter.NULL);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
    ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output =
            new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();

    mapper.map(new ChukwaArchiveKey(), chunk, output, Reporter.NULL);
    ChukwaRecordKey recordKey = new ChukwaRecordKey("someReduceType", SAMPLE_RECORD_DATA);

    assertEquals("MockMapProcessor never invoked - no records found", 1, output.data.size());
    assertNotNull("MockMapProcessor never invoked", output.data.get(recordKey));
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
    ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output =
            new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();

    mapper.map(new ChukwaArchiveKey(), chunk, output, Reporter.NULL);
    ChukwaRecordKey recordKey = new ChukwaRecordKey("someReduceType", SAMPLE_RECORD_DATA);

    assertEquals("MockMapProcessor never invoked - no records found", 1, output.data.size());
    assertNotNull("MockMapProcessor never invoked", output.data.get(recordKey));
  }
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
    }
    long currentTimeMillis = System.currentTimeMillis();
    boolean isSuccessful = true;
    String recordType = null;

    ChukwaRecordKey key = new ChukwaRecordKey();
    ChukwaRecord record = new ChukwaRecord();
    String cluster = null;
    int numOfRecords = 0;
    try {
      Pattern p = Pattern.compile("(.*)\\-(\\d+)$");
      int batch = 0;
      while (reader.next(key, record)) {
      numOfRecords++;
        if(first) {
          try {
            cluster = RecordUtil.getClusterName(record);
            initEnv(cluster);
            first=false;
          } catch(Exception ex) {
            log.error("Initialization failed for: "+cluster+".  Please check jdbc configuration.");
            return false;
          }
        }
        String sqlTime = DatabaseWriter.formatTimeStamp(record.getTime());
        log.debug("Timestamp: " + record.getTime());
        log.debug("DataType: " + key.getReduceType());

        String[] fields = record.getFields();
        String table = null;
        String[] priKeys = null;
        HashMap<String, HashMap<String, String>> hashReport = new HashMap<String, HashMap<String, String>>();
        StringBuilder normKey = new StringBuilder();
        String node = record.getValue("csource");
        recordType = key.getReduceType().toLowerCase();
        String dbKey = "report.db.name." + recordType;
        Matcher m = p.matcher(recordType);
        if (dbTables.containsKey(dbKey)) {
          String tableName = mdlConfig.get(dbKey);
          if (!RegexUtil.isRegex(tableName)) {
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
    TsProcessor p = new TsProcessor();
    p.reset(chunk);
    p.process(null, chunk, output, Reporter.NULL);

    ChukwaRecordKey key = buildKey(date, DATA_SOURCE, DATA_TYPE);
    Map<ChukwaRecordKey, ChukwaRecord> outputData = output.data;

    assertNotNull("No output data found.", outputData);
    assertEquals("Output data size not correct.", 1, outputData.size());
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    calendar.setTime(date);
    calendar.set(Calendar.MINUTE, 0);
    calendar.set(Calendar.SECOND, 0);
    calendar.set(Calendar.MILLISECOND, 0);

    ChukwaRecordKey key = new ChukwaRecordKey();
    key.setKey("" + calendar.getTimeInMillis() + "/" + dataSource + "/" + date.getTime());
    key.setReduceType(dataType);

    return key;
  }
View Full Code Here
179
180
181
182
183
184
185
186
187
            new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();

    p.process(null, chunk, output, Reporter.NULL);

    assertEquals("Output data size not correct.", 1, output.data.size());
    ChukwaRecordKey key = output.data.keySet().iterator().next();
    ChukwaRecord record = output.data.get(key);
    assertNull("Output should not be error.", record.getValue("cchunkData"));
  }
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
    // Hard-coded to check the contents of test/samples/JobLog.log
    try {

      Pattern task_id_pat = Pattern.compile("attempt_[0-9]*_[0-9]*_[mr]_([0-9]*)_[0-9]*");

      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();

      // initialize data structures for checking FSM
      // should see 10 maps, 8 reduces
      boolean mapSeen[] = new boolean[10];
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
        fsmSink + "*/*/*/*.evt");
      FileStatus [] files;
      files = fileSys.globStatus(fsm_outputs);
      int count = 0;
      int numHDFSRead = 0, numHDFSWrite = 0, numShuffles = 0;
      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();

      for (int i = 0; i < files.length; i++) {
        SequenceFile.Reader r = new SequenceFile.Reader(fileSys, files[i].getPath(), conf);
        System.out.println("Processing files " + files[i].getPath().toString());
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    }
    long currentTimeMillis = System.currentTimeMillis();
    boolean isSuccessful = true;
    String recordType = null;

    ChukwaRecordKey key = new ChukwaRecordKey();
    ChukwaRecord record = new ChukwaRecord();
    String cluster = null;
    int numOfRecords = 0;
    try {
      Pattern p = Pattern.compile("(.*)\\-(\\d+)$");
      int batch = 0;
      while (reader.next(key, record)) {
      numOfRecords++;
        if(first) {
          try {
            cluster = RecordUtil.getClusterName(record);
            initEnv(cluster);
            first=false;
          } catch(Exception ex) {
            log.error("Initialization failed for: "+cluster+".  Please check jdbc configuration.");
            return false;
          }
        }
        String sqlTime = DatabaseWriter.formatTimeStamp(record.getTime());
        log.debug("Timestamp: " + record.getTime());
        log.debug("DataType: " + key.getReduceType());

        String[] fields = record.getFields();
        String table = null;
        String[] priKeys = null;
        HashMap<String, HashMap<String, String>> hashReport = new HashMap<String, HashMap<String, String>>();
        StringBuilder normKey = new StringBuilder();
        String node = record.getValue("csource");
        recordType = key.getReduceType().toLowerCase();
        String dbKey = "report.db.name." + recordType;
        Matcher m = p.matcher(recordType);
        if (dbTables.containsKey(dbKey)) {
          String[] tmp = mdlConfig.findTableName(mdlConfig.get(dbKey), record
              .getTime(), record.getTime());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.ChukwaStorage
  • org.apache.hadoop.chukwa.dataloader.MetricDataLoader
  • org.apache.hadoop.chukwa.extraction.database.MetricDataLoader
  • org.apache.hadoop.chukwa.extraction.database.MRJobCounters
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ChunkSaver
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Df
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HiTune.Instrument
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HiTune.JobHistoryLog
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HiTune.Sysstat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.