Package com.linkedin.whiteelephant.parsing

Examples of com.linkedin.whiteelephant.parsing.LogData


        {
          task = LineParsing.tryParseTask(line);
        }
      }
     
      LogData data = new LogData();
      String jobId = null;
     
      data.setCluster(_clusterName);
     
      try
      {
        if (job != null)
        {
          // log lines are sometimes truncated, so data may be missing - just ignore these
          if (job.getJobId() != null)
          {
            jobId = job.getJobId().toString();
            job.setTasks(new ArrayList<Task>());
            data.setEntry(job);
            data.setPath(findInputSplitForJob(jobId,inputSplits));
            context.write(new AvroKey<String>(jobId), new AvroValue<LogData>(data));
          }
        }
        else if (attempt != null)
        {
          // log lines are sometimes truncated, so data may be missing - just ignore these
          if (attempt.getJobId() != null && attempt.getTaskId() != null && attempt.getTaskAttemptId() != null && attempt.getType() != null)
          {
            jobId = attempt.getJobId().toString();
            data.setEntry(attempt);
            data.setPath(findInputSplitForJob(jobId,inputSplits));
            context.write(new AvroKey<String>(jobId), new AvroValue<LogData>(data));
          }
        }
        else if (task != null)
        {
          // log lines are sometimes truncated, so data may be missing - just ignore these
          if (task.getJobId() != null && task.getTaskId() != null && task.getType() != null)
          {
            jobId = task.getJobId().toString();
            task.setAttempts(new ArrayList<Attempt>());
            data.setEntry(task);
            data.setPath(findInputSplitForJob(jobId,inputSplits));
            context.write(new AvroKey<String>(jobId), new AvroValue<LogData>(data));
          }
        }
      }
      catch (Exception e)
      {
        System.out.println("Exception writing log data: " + e.toString());
        if (jobId != null)
        {
          System.out.println("jobId: " + jobId);
          CharSequence path = data.getPath();
          if (path != null)
          {
            System.out.println("input: " + path);
          }
        }
View Full Code Here


     
      mergeJobEntries(job, jobEntries);
      mergeTaskEntries(job, taskEntries);
      mergeTaskAttemptEntries(job, attemptEntries);     
     
      LogData data = new LogData();
      data.setPath(inputPath);
      data.setEntry(job);
     
      try
      {
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MMdd");    
        dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
View Full Code Here

    }
   
    @Override
    protected void map(AvroKey<String> key, AvroValue<LogData> value, Context context) throws java.io.IOException, java.lang.InterruptedException
    {
      LogData data = value.datum();
     
      if (data.getEntry() != null && data.getEntry() instanceof com.linkedin.whiteelephant.parsing.Job)
      {
        com.linkedin.whiteelephant.parsing.Job job = (com.linkedin.whiteelephant.parsing.Job)data.getEntry();
        for (com.linkedin.whiteelephant.parsing.Task task : job.getTasks())
        {
          for (com.linkedin.whiteelephant.parsing.Attempt attempt : task.getAttempts())
          {
            if (attempt.getTaskStatus() == null)
View Full Code Here

TOP

Related Classes of com.linkedin.whiteelephant.parsing.LogData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.