Package org.apache.hadoop.chukwa.extraction.database

Examples of org.apache.hadoop.chukwa.extraction.database.DatabaseHelper

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
        String body = matcher.group(4);
       
         String[] kvpairs = body.split(" ");
           
        // database
        DatabaseHelper databaseRecord = new DatabaseHelper("HadoopMetrics");
       
          for(int i = 1 ; i < kvpairs.length; ++i)
          {
            String kvpair =  kvpairs[i];
            String[] halves = kvpair.split("=");
            if(halves[0].equals("chukwa_timestamp"))
            {
              key.set(halves[1]);
            }
            else
            {
              databaseRecord.add(d.getTime(), halves[0], halves[1]);
            }
          }
         
          //Output NodeActivity info to database
        output.collect(key, databaseRecord.buildChukwaRecord());
        log.info("HadoopMetricsProcessor output 1 Hadoop's Metric to database");
      }
      catch (ParseException e)
      {
        e.printStackTrace();
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
          log.debug("skip:"+lines[i]);
            i++;
        }
        while (i < lines.length)
        {
          DatabaseHelper databaseRecord = null;
          if(lines[i].equals("")) {
            i++;
            headers = parseHeader(lines[i]);
            i++;
          }
          String data[] = parseData(lines[i]);
          if(headers[1].equals("IFACE") && headers[2].equals("rxpck/s")) {
            log.debug("Matched Sar-Network");
            databaseRecord = new DatabaseHelper("system");
          } else if(headers[1].equals("IFACE") && headers[2].equals("rxerr/s")) {
            log.debug("Matched Sar-Network");
            databaseRecord = new DatabaseHelper("system"); 
          } else if(headers[1].equals("kbmemfree")) {
            log.debug("Matched Sar-Memory");
            databaseRecord = new DatabaseHelper("system"); 
          } else if(headers[1].equals("totsck")) {
            log.debug("Matched Sar-NetworkSockets");
            databaseRecord = new DatabaseHelper("system"); 
          } else if(headers[1].equals("runq-sz")) {
            log.debug("Matched Sar-LoadAverage");
            databaseRecord = new DatabaseHelper("system"); 
          } else {
            log.debug("No match:"+headers[1]+" "+headers[2]);
          }
          if(databaseRecord!=null) {
            int j=0;
            log.debug("Data Length: " + data.length);
                      while(j<data.length) {
                        log.debug("header:"+headers[j]+" data:"+data[j]);
                        if(!headers[j].equals("Average:")) {
                    databaseRecord.add(d.getTime(),headers[j],data[j]);
                        }
                j++;
                      }           
            //Output Sar info to database
            output.collect(key, databaseRecord.buildChukwaRecord());
          }
          i++;
        }
        // End of parsing
      } catch (Exception e)
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
          throw new DFInvalidRecord("Wrong output format (header) [" + recordEntry + "]");
        }
       
        String[] values = null;
        // database (long timestamp,String table,String tag,int tableType,int sqlType)
        DatabaseHelper databaseRecord = new DatabaseHelper("system");
       
        // Data
        //databaseRecord.addKey("Used",""+totalUsedNode);
       
        for (int i=1;i<lines.length;i++)
        {
          values = lines[i].split("[\\s]++");
          databaseRecord.add(d.getTime(),values[0]+"."+columns[1], values[1]);
          databaseRecord.add(d.getTime(),values[0]+"."+columns[2], values[2]);
          databaseRecord.add(d.getTime(),values[0]+"."+columns[3], values[3]);
          databaseRecord.add(d.getTime(),values[0]+"."+columns[4], values[4]);
          databaseRecord.add(d.getTime(),values[0]+"."+columns[5], values[5]);
        }
        //Output DF info to database
        output.collect(key, databaseRecord.buildChukwaRecord());
        log.info("DFProcessor output 1 DF to database");
      }
      catch (ParseException e)
      {
        e.printStackTrace();
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        }
       
        // End of parsing

        // database (long timestamp,String table,String tag,int tableType,int sqlType)
        DatabaseHelper databaseRecord = new DatabaseHelper("NodeActivity");
       
        // Data
        databaseRecord.add(d.getTime(),"Used",""+totalUsedNode);
        databaseRecord.add(d.getTime(),"Free",""+totalFreeNode);
        databaseRecord.add(d.getTime(),"Down",""+totalDownNode);
       
        //Output NodeActivity info to database
        output.collect(key, databaseRecord.buildChukwaRecord());
        log.info("PbsNodeProcessor output 1 NodeActivity to database");
       
        // INFO if you need to save NodeActivity info to HDFS
        // use the following block
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
        try
        {
          JSONObject json = new JSONObject(body);
         
          // database
          DatabaseHelper databaseRecord = new DatabaseHelper("switches");//
          String tag = json.getString("poller") + "-" + json.getString("host");
          String metricName = json.getString("metricName");
         
          // Data
          JSONObject jsonData = json.getJSONObject("data").getJSONObject("data");
         
       
          long ts = 0;
          double value = 0;
          String jsonTs = null;
          String jsonValue = null;
          Iterator<String> it = jsonData.keys();
          while(it.hasNext())
          {
            jsonTs = it.next();
            jsonValue = jsonData.getString(jsonTs);
            try
            {
              value = Double.parseDouble(jsonValue);
              ts = Long.parseLong(jsonTs);
              databaseRecord.add(ts,metricName,value);
            }
            catch(NumberFormatException e)
            {
              if (!jsonValue.equalsIgnoreCase("N/A"))
              {
                throw new YwatchInvalidEntry("YWatchProcessor invalid entry [" + body + "]");
              }
            }
          }
         
          output.collect(key, databaseRecord.buildChukwaRecord());
          log.info("YWatchProcessor output 1 metric to database");
         
        }
        catch (IOException e)
        {
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
          }
          i++;         
        }
        while (i < lines.length)
        {
          DatabaseHelper databaseRecord = null;
          if(lines[i].equals("")) {
            i++;
            headers = parseHeader(lines[i]);
            i++;
          }
          String data[] = parseData(lines[i]);
          if(headers[0].equals("avg-cpu:")) {
            log.debug("Matched CPU-Utilization");
            databaseRecord = new DatabaseHelper("system");
          } else if(headers[0].equals("Device:")) {
            log.debug("Matched Iostat");
            databaseRecord = new DatabaseHelper("system"); 
          } else {
            log.debug("No match:"+headers[0]);
          }
          if(databaseRecord!=null) {
            int j=0;
            log.debug("Data Length: " + data.length);
                      while(j<data.length) {
                        log.debug("header:"+headers[j]+" data:"+data[j]);
                        if(!headers[j].equals("avg-cpu:")) {
                    databaseRecord.add(d.getTime(),headers[j],data[j]);
                        }
                j++;
                      }           
            //Output Sar info to database
            output.collect(key, databaseRecord.buildChukwaRecord());
          }
          i++;
        }
        // End of parsing
      } catch (Exception e)
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
            i++;
        }
        i++;
        String[] headers = lines[i].split("\\s+");
        HashMap<String, String>summary = parseSummary(summaryString);
        DatabaseHelper databaseRecord = new DatabaseHelper("system");
        Iterator<String> ki = summary.keySet().iterator();
        while(ki.hasNext()) {
          String key = ki.next();
            databaseRecord.add(d.getTime(),key, summary.get(key));
        }
        output.collect(key, databaseRecord.buildChukwaRecord());
        while (i < lines.length)
        {
          databaseRecord = null;
          String data[] = lines[i].split("\\s+",headers.length);
          if(lines[i].indexOf("PID USER")<0) {
            databaseRecord = new DatabaseHelper("system"); 
          }
          if(databaseRecord!=null) {
            int j=0;
            log.debug("Data Length: " + data.length);
                      while(j<data.length-1) {
                        if(headers[0].equals("")) {
                          log.debug("header:"+headers[j+1]+" data:"+data[j+1]);
                          databaseRecord.add(d.getTime(),headers[j+1],data[j+1]);
                        } else {
                          log.debug("header:"+headers[j+1]+" data:"+data[j]);
                  databaseRecord.add(d.getTime(),headers[j+1],data[j]);
                        }
                j++;
                      }           
            //Output Sar info to database
            output.collect(key, databaseRecord.buildChukwaRecord());
          }
          i++;
        }
        // End of parsing
      } catch (Exception e)
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.extraction.database.DatabaseHelper

  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.DFProcessor
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Iostat
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.PbsNodesProcessor
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Sar
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Top
  • org.apache.hadoop.chukwa.extraction.demux.processor.mapper.YWatchProcessor
  • org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.