Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord

Package org.apache.hadoop.chukwa.extraction.engine

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord

org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord

    log.info("File: [" +  file + "]" + fs.exists(new Path(file)));
    try {
      reader = new SequenceFile.Reader(fs, new Path(file), conf);


      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();


      StringBuilder sb = new StringBuilder();
      while (reader.next(key, record)) {
       
        sb.append("===== KEY   =====");


        sb.append("DataType: " + key.getReduceType());
        sb.append("Key: " + key.getKey());
        sb.append("===== Value =====");


        String[] fields = record.getFields();
        Arrays.sort(fields );
        sb.append("Timestamp : " + record.getTime());
        for (String field : fields) {
          sb.append("[" + field + "] :" + record.getValue(field));
        }
      }
      
      return sb.toString();
    } catch (Throwable e) {

View Full Code Here

      machine = "M0";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "7");
      record.add("B", "3");
      record.add("C", "9");
  
      seqFileWriter.append(key, record);
    }


    {    
      machine = "M0";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("D", "1");
  
      seqFileWriter.append(key, record);
    }


    {    
      machine = "M1";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "17");
  
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("B", "37");
      record.add("C", "51");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205860; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("C", "3");
      record.add("D", "12");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205860; // Wed, 13 May 2009 09:11:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("B", "6");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205860; // Wed, 13 May 2009 09:11:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "13.2");
      record.add("B", "23");
      record.add("C", "8.5");
      record.add("D", "6");
      
      // create duplicate
      seqFileWriter.append(key, record);
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("B", "6");
      record.add("C", "8");
      record.add("D", "6");
      record.add("E", "48.5");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8.3");
      record.add("B", "5.2");
      record.add("C", "37.7");
      record.add("D", "61.9");
      record.add("E", "40.3");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205980; // Wed, 13 May 2009 09:13:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "18.3");
      record.add("B", "1.2");
      record.add("C", "7.7");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M2";
      long time = 1242205980; // Wed, 13 May 2009 09:13:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8.9");
      record.add("B", "8.3");
      record.add("C", "7.2");
      record.add("D", "6.1");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M3";
      // late arrival T0
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "12.5");
      record.add("B", "26.82");
      record.add("C", "89.51");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M4";
      // late arrival T0
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "13.91");
      record.add("B", "21.02");
      record.add("C", "18.05");
      seqFileWriter.append(key, record);
    }
    
    seqFileWriter.close();
  }

View Full Code Here

      String jobID = "";
        matcher = jobPattern.matcher(capp);
        if(matcher.matches()) {
          jobID=matcher.group(2);
        }
        ChukwaRecord record = new ChukwaRecord();
        ChukwaRecord jobConfRecord = new ChukwaRecord();
      DocumentBuilderFactory docBuilderFactory 
        = DocumentBuilderFactory.newInstance();
      //ignore all comments inside the xml file
      docBuilderFactory.setIgnoringComments(true);
      try {
          DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
          Document doc = null;
          String fileName = "test_"+randomNumber.nextInt();
          File tmp = new File(fileName);
          FileOutputStream out = new FileOutputStream(tmp);
          out.write(recordEntry.getBytes());
          out.close();
        doc = builder.parse(fileName);
        Element root = doc.getDocumentElement();
        if (!"configuration".equals(root.getTagName()))
            log.fatal("bad conf file: top-level element not <configuration>");
        NodeList props = root.getChildNodes();
            JSONObject json = new JSONObject();
            String queue = "default";
    
        for (int i = 0; i < props.getLength(); i++) {
            Node propNode = props.item(i);
            if (!(propNode instanceof Element))
                continue;
            Element prop = (Element)propNode;
            if (!"property".equals(prop.getTagName()))
                log.warn("bad conf file: element not <property>");
            NodeList fields = prop.getChildNodes();
            String attr = null;
            String value = null;
            boolean finalParameter = false;
            for (int j = 0; j < fields.getLength(); j++) {
                Node fieldNode = fields.item(j);
                if (!(fieldNode instanceof Element))
                    continue;
                Element field = (Element)fieldNode;
                if ("name".equals(field.getTagName()) && field.hasChildNodes())
                    attr = ((Text)field.getFirstChild()).getData().trim();
                if ("value".equals(field.getTagName()) && field.hasChildNodes())
                    value = ((Text)field.getFirstChild()).getData();
                if ("final".equals(field.getTagName()) && field.hasChildNodes())
                    finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
            }
            
            // Ignore this parameter if it has already been marked as 'final'
            if (attr != null && value != null) {
                json.put(attr, value);
                if(attr.intern()=="mapred.job.queue.name".intern()) {
                    queue=value;
                }
                jobConfRecord.add("job_conf." + attr, value);
            }
        }
        record.add("JOBCONF-JSON", json.toString());
        record.add("mapred.job.queue.name", queue);
        record.add("JOBID", "job_" + jobID);
        buildGenericRecord(record, null, time, "JobData");
        calendar.setTimeInMillis(time);
        calendar.set(Calendar.MINUTE, 0);
        calendar.set(Calendar.SECOND, 0);
        calendar.set(Calendar.MILLISECOND, 0);
        key.setKey("" + calendar.getTimeInMillis() + "/job_" + jobID + "/" + time);
        output.collect(key, record);


        jobConfRecord.add("JOBID", "job_" + jobID);
        buildGenericRecord(jobConfRecord, null, time, "JobConfData");
        output.collect(key, jobConfRecord);
            
        tmp.delete();
      } catch(Exception e) {

View Full Code Here

      }
      kvMatcher.reset(recordEntry);
      if (!kvMatcher.find()) {
        throw new IOException("Failed to find record");
      }
      ChukwaRecord rec = new ChukwaRecord();
      do {
        rec.add(kvMatcher.group(1), kvMatcher.group(2));
      } while (kvMatcher.find());
      Locality loc = getLocality(rec.getValue("src"), rec.getValue("dest"));
      rec.add("locality", loc.getLabel());


      calendar.setTimeInMillis(ms);
      calendar.set(Calendar.SECOND, 0);
      calendar.set(Calendar.MILLISECOND, 0);
      ms = calendar.getTimeInMillis();
      calendar.set(Calendar.MINUTE, 0);
      key.setKey(calendar.getTimeInMillis() + "/" + loc.getLabel() + "/" +
                 rec.getValue("op").toLowerCase() + "/" + ms);
      key.setReduceType("ClientTrace");
      rec.setTime(ms);


      rec.add(Record.tagsField, chunk.getTags());
      rec.add(Record.sourceField, chunk.getSource());
      rec.add(Record.applicationField, chunk.getApplication());
      rec.add("actual_time",(new Long(ms_fullresolution)).toString());
      output.collect(key, rec);


    } catch (ParseException e) {
      log.warn("Unable to parse the date in DefaultProcessor ["
          + recordEntry + "]", e);

View Full Code Here

             rs = stmt.getResultSet();
             ResultSetMetaData rmeta = rs.getMetaData();
             int col = rmeta.getColumnCount();
             while (rs.next()) 
             {
               ChukwaRecord event = new ChukwaRecord();
             String cell="";
             long timestamp = 0;
             
             for(int i=1;i<col;i++)
             {
                 String value = rs.getString(i);
                 if(value!=null) 
                 {
               cell=cell+" "+rmeta.getColumnName(i)+":"+value;
                 }
                 if(rmeta.getColumnName(i).equals(timeField)) 
                 {
                   timestamp = rs.getLong(i);
                   event.setTime(timestamp);
                 }
             }
             boolean isValid = false;
             if(filter == null || filter.equals("")) 
             {
               isValid = true;
             }
             else if (cell.indexOf(filter) > 0)
             {
               isValid = true;
             }
             if (!isValid)
             { continue; }
             
             event.add(Record.bodyField, cell);
             event.add(Record.sourceField, cluster + "." + dataSource );
             if (records.containsKey(timestamp))
             {
               records.get(timestamp).add(event);
             }
             else
             {
               List<Record> list = new LinkedList<Record>();
               list.add(event);
               records.put(event.getTime(), list);
             }     
             }
         }
      }
      catch (SQLException e)

View Full Code Here

      batch.append(keyValues);
    }    
        return batch.toString();
  }
  public ChukwaRecord buildChukwaRecord()  {
    ChukwaRecord chukwaRecord = new ChukwaRecord();
    StringBuilder queries = new StringBuilder();
    
    chukwaRecord.add(sqlField, this.toString());
    chukwaRecord.add(Record.destinationField, DatabaseHelper.databaseRecordType + table + ".evt");
    chukwaRecord.add(Record.dataSourceField, DatabaseHelper.databaseRecordType + table );
    return chukwaRecord;
  }

View Full Code Here

        {
          fs.rename(datasourceDirectory.getPath(), inProgressdatabaseInputFilePath);


          SequenceFile.Reader r = new SequenceFile.Reader(fs,datasourceDirectory.getPath(), conf);
          Text key = new Text();
          ChukwaRecord record = new ChukwaRecord();
          try
          {
            while (r.next(key, record))
            {
              System.out.println(record.getValue(DatabaseHelper.sqlField));
            } // End while(r.next(key, databaseRecord) )
          } // end Try
          catch (Exception e)
          {
            log.error("Unable to insert data into database"

View Full Code Here

               }
             
             if (valid)
             {
//              System.out.println("In Range In Filter: " + line);
              ChukwaRecord record = new ChukwaRecord();
              record.setTime(timestamp);
              record.add("offset", ""+offset);
              record.add(Record.bodyField, data[1]);
              record.add(Record.sourceField, dataSource);
              
              records.add(record);
              listSize = records.size();
              if (listSize > maxRows)
              {

View Full Code Here

      long offset = 0;
      
//      HdfsWriter.HdfsWriterKey key = new HdfsWriter.HdfsWriterKey();
      Text key = new Text();


        ChukwaRecord evt = new ChukwaRecord();
      while(r.next(key, evt))
      {  
        lineCount ++;
        
        System.out.println("NameNodeParser Line [" +evt.getValue(Record.bodyField) + "]");  
        
        if (evt != null)
        {
          timestamp = evt.getTime();
          if (timestamp < t0) 
          {
             System.out.println("Line not in range. Skipping: " +evt.getValue(Record.bodyField));
             System.out.println("Search for: " + new Date(t0) + " is :" + new Date(timestamp));
             continue;
          } 
          else if ((timestamp < t1) && (offset < maxOffset )) //JB (epochTS < maxDate)
          {
            
            System.out.println("In Range: " + evt.getValue(Record.bodyField));
            boolean valid = false;
            
             if ( (filter == null || filter.equals("") ))
             {
               valid = true;
             }
             else if (evt.getValue(Record.rawField).toLowerCase().indexOf(filter) > 0)
               {
 System.out.println("MATCH " +  filter + "===========================>>>>>>>" + evt.getValue(Record.rawField));
                 valid = true;
               }
             
             if (valid)
             {
              records.add(evt);
evt = new ChukwaRecord();
              listSize = records.size();
              if (listSize > maxRows)
              {
                records.remove(0);
                System.out.println("==========>>>>>REMOVING: " + evt.getValue(Record.bodyField));
              }
             }
            else 
            {
              System.out.println("In Range ==================>>>>>>>>> OUT Regex: " + evt.getValue(Record.bodyField));
            }


          }
          else
          {
             System.out.println("Line out of range. Stopping now: " +evt.getValue(Record.bodyField));
            break;
          }
        }


      }

View Full Code Here

                        args[0], fs, conf);
      
      SequenceFile.Reader r= new SequenceFile.Reader(fs, new Path(args[0]), conf);
      Text key = new Text();
      
      ChukwaRecord evt = new ChukwaRecord();
      while(r.next(key, evt))
      {
        System.out.println( evt);
      }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord

edu.berkeley.chukwa_xtrace.XtrExtract$MapClass

org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder

org.apache.hadoop.chukwa.ChukwaStorage

org.apache.hadoop.chukwa.datacollection.writer.test.demux.TextParser

org.apache.hadoop.chukwa.dataloader.MetricDataLoader

org.apache.hadoop.chukwa.extraction.database.DatabaseHelper

org.apache.hadoop.chukwa.extraction.database.DatabaseLoader

org.apache.hadoop.chukwa.extraction.database.MetricDataLoader

org.apache.hadoop.chukwa.extraction.database.MRJobCounters

org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ChunkSaver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.