Examples of com.linkedin.whiteelephant.parsing.Attempt

Package com.linkedin.whiteelephant.parsing

Examples of com.linkedin.whiteelephant.parsing.Attempt

com.linkedin.whiteelephant.parsing.Attempt

      }
      
      String line = value.toString(); 
      
      Job job = null;
      Attempt attempt = null;
      Task task = null;
      
      job = LineParsing.tryParseJob(line);
      
      if (job == null)
      {      
        attempt = LineParsing.tryParseAttempt(line);
        
        if (attempt == null)
        {
          task = LineParsing.tryParseTask(line);
        }
      }
      
      LogData data = new LogData();
      String jobId = null;
      
      data.setCluster(_clusterName);
      
      try
      {
        if (job != null)
        {
          // log lines are sometimes truncated, so data may be missing - just ignore these
          if (job.getJobId() != null)
          {
            jobId = job.getJobId().toString();
            job.setTasks(new ArrayList<Task>());
            data.setEntry(job);
            data.setPath(findInputSplitForJob(jobId,inputSplits));
            context.write(new AvroKey<String>(jobId), new AvroValue<LogData>(data));
          }
        }
        else if (attempt != null)
        {
          // log lines are sometimes truncated, so data may be missing - just ignore these
          if (attempt.getJobId() != null && attempt.getTaskId() != null && attempt.getTaskAttemptId() != null && attempt.getType() != null)
          {
            jobId = attempt.getJobId().toString();
            data.setEntry(attempt);
            data.setPath(findInputSplitForJob(jobId,inputSplits));
            context.write(new AvroKey<String>(jobId), new AvroValue<LogData>(data));
          }
        }

View Full Code Here

    {
      // merge together the entries for each task attempt
      Map<String,Attempt> taskAttemptIdToAttempt = new HashMap<String,Attempt>();      
      for (Attempt attempt : attemptEntries)
      {        
        Attempt mergedAttempt;
        if (!taskAttemptIdToAttempt.containsKey(attempt.getTaskAttemptId().toString()))
        {
          mergedAttempt = new Attempt();
          mergedAttempt.setCounters(new HashMap<CharSequence,Long>());
          mergedAttempt.setDerived(new DerivedAttemptData());
          taskAttemptIdToAttempt.put(attempt.getTaskAttemptId().toString(),mergedAttempt);
        }
        else
        {
          mergedAttempt = taskAttemptIdToAttempt.get(attempt.getTaskAttemptId().toString());
        }              
        
        if (attempt.getType() != null)
        {
          mergedAttempt.setType(attempt.getType());
        }
        
        if (attempt.getJobId() != null)
        {
          mergedAttempt.setJobId(attempt.getJobId());
        }
        
        if (attempt.getTaskId() != null)
        {
          mergedAttempt.setTaskId(attempt.getTaskId());
        }
        
        if (attempt.getTaskAttemptId() != null)
        {
          mergedAttempt.setTaskAttemptId(attempt.getTaskAttemptId());
        }
        
        if (attempt.getStartTime() != null)
        {
          // take the later start time in case there are multiple
          if (mergedAttempt.getStartTime() == null || mergedAttempt.getStartTime() < attempt.getStartTime())
          {
            mergedAttempt.setStartTime(attempt.getStartTime());
          }
        }
        
        if (attempt.getFinishTime() != null)
        {
          // take the later finish time in case there are multiple
          if (mergedAttempt.getFinishTime() == null || mergedAttempt.getFinishTime() < attempt.getFinishTime())
          {
            mergedAttempt.setFinishTime(attempt.getFinishTime());
          }
        }
        
        if (attempt.getShuffleFinished() != null)
        {
          // take the later finish time in case there are multiple
          if (mergedAttempt.getShuffleFinished() == null || mergedAttempt.getShuffleFinished() < attempt.getShuffleFinished())
          {
            mergedAttempt.setShuffleFinished(attempt.getShuffleFinished());
          }
        }
        
        if (attempt.getSortFinished() != null)
        {
          // take the later finish time in case there are multiple
          if (mergedAttempt.getSortFinished() == null || mergedAttempt.getSortFinished() < attempt.getSortFinished())
          {
            mergedAttempt.setSortFinished(attempt.getSortFinished());
          }
        }
                
        if (attempt.getTaskStatus() != null)
        {
          mergedAttempt.setTaskStatus(attempt.getTaskStatus());
        }
        
        if (attempt.getCounters() != null && attempt.getCounters().size() > 0)
        {
          mergedAttempt.setCounters(attempt.getCounters());
        }
      }
      
      // filter out bad data
      Collection<Attempt> filteredAttempts = Collections2.filter(taskAttemptIdToAttempt.values(), new Predicate<Attempt>() {

View Full Code Here

  public static Attempt tryParseAttempt(String line)
  {
    // these mess with our pattern matching
    line = line.replace("\\\"", "");
    
    Attempt attempt = null;
    
    Matcher m = attemptLinePattern.matcher(line);
    
    if (m.matches())
    {
      attempt = new Attempt();
            
      attempt.setCounters(new HashMap<CharSequence,Long>());
      
      attempt.setDerived(new DerivedAttemptData());
      
      attempt.setType(TaskType.valueOf(m.group(1).toUpperCase()));
      
      Matcher matcher = parameterPattern.matcher(line);
      
      while (matcher.find())
      {
        String name = matcher.group(1);
        String value = matcher.group(2);
        maybeSetAttemptParam(attempt,name,value);
      }
      
      matcher = counterPattern.matcher(line);
      
      while (matcher.find())
      {
        String name = matcher.group(1);
        String value = matcher.group(2);
        setAttemptCounter(attempt,name,Long.parseLong(value));        
      }
      
      if (attempt.getTaskId() != null)
      {
        Matcher taskMatcher = taskPattern.matcher(attempt.getTaskId());
        
        if (taskMatcher.matches())
        {
          String jobId = String.format("job_%s",taskMatcher.group(1));
          attempt.setJobId(jobId);
        }
        else
        {
          System.out.println("Could not match task ID for " + attempt.getTaskId());
          System.out.println("line: " + line);
        }
      }
      else
      {

View Full Code Here

TOP

Related Classes of com.linkedin.whiteelephant.parsing.Attempt

com.linkedin.whiteelephant.parsing.LineParsing

com.linkedin.whiteelephant.parsing.ParseJobsFromLogs$TheMapper

com.linkedin.whiteelephant.parsing.ParseJobsFromLogs$TheReducer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.