Package org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat

Examples of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey


  private int dumpAContainerLogs(String containerIdStr,
      AggregatedLogFormat.LogReader reader, PrintStream out)
      throws IOException {
    DataInputStream valueStream;
    LogKey key = new LogKey();
    valueStream = reader.next(key);

    while (valueStream != null && !key.toString().equals(containerIdStr)) {
      // Next container
      key = new LogKey();
      valueStream = reader.next(key);
    }

    if (valueStream == null) {
      System.out.println("Logs for container " + containerIdStr
View Full Code Here


          new AggregatedLogFormat.LogReader(getConf(),
              new Path(remoteAppLogDir, thisNodeFile.getPath().getName()));
      try {

        DataInputStream valueStream;
        LogKey key = new LogKey();
        valueStream = reader.next(key);

        while (valueStream != null) {
          String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
          out.println(containerString);
          out.println(StringUtils.repeat("=", containerString.length()));
          while (true) {
            try {
              LogReader.readAContainerLogsForALogType(valueStream, out);
            } catch (EOFException eof) {
              break;
            }
          }

          // Next container
          key = new LogKey();
          valueStream = reader.next(key);
        }
      } finally {
        reader.close();
      }
View Full Code Here

    try {
      Map<String, Map<String, String>> logMap =
          new HashMap<String, Map<String, String>>();
      DataInputStream valueStream;

      LogKey key = new LogKey();
      valueStream = reader.next(key);

      while (valueStream != null) {
        LOG.info("Found container " + key.toString());
        Map<String, String> perContainerMap = new HashMap<String, String>();
        logMap.put(key.toString(), perContainerMap);

        while (true) {
          try {
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintStream ps = new PrintStream(baos);
            LogReader.readAContainerLogsForALogType(valueStream, ps);

            String writtenLines[] = baos.toString().split(
              System.getProperty("line.separator"));

            Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
            String fileType = writtenLines[0].substring(9);

            Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
            String fileLengthStr = writtenLines[1].substring(11);
            long fileLength = Long.parseLong(fileLengthStr);

            Assert.assertEquals("Log Contents:",
              writtenLines[2].substring(0, 13));

            String logContents = StringUtils.join(
              Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
            perContainerMap.put(fileType, logContents);

            LOG.info("LogType:" + fileType);
            LOG.info("LogType:" + fileLength);
            LOG.info("Log Contents:\n" + perContainerMap.get(fileType));
          } catch (EOFException eof) {
            break;
          }
        }

        // Next container
        key = new LogKey();
        valueStream = reader.next(key);
      }

      // 1 for each container
      Assert.assertEquals(expectedContainerIds.length, logMap.size());
View Full Code Here

  @Private
  public int dumpAContainerLogs(String containerIdStr,
      AggregatedLogFormat.LogReader reader, PrintStream out) throws IOException {
    DataInputStream valueStream;
    LogKey key = new LogKey();
    valueStream = reader.next(key);

    while (valueStream != null && !key.toString().equals(containerIdStr)) {
      // Next container
      key = new LogKey();
      valueStream = reader.next(key);
    }

    if (valueStream == null) {
      System.out.println("Logs for container " + containerIdStr
View Full Code Here

      AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(
          getConf(), new Path(remoteAppLogDir, thisNodeFile.getPath().getName()));
      try {

        DataInputStream valueStream;
        LogKey key = new LogKey();
        valueStream = reader.next(key);

        while (valueStream != null) {
          String containerString = "\n\nContainer: " + key + " on "
              + thisNodeFile.getPath().getName();
          out.println(containerString);
          out.println(StringUtils.repeat("=", containerString.length()));
          while (true) {
            try {
              LogReader.readAContainerLogsForALogType(valueStream, out);
            } catch (EOFException eof) {
              break;
            }
          }

          // Next container
          key = new LogKey();
          valueStream = reader.next(key);
        }
      } finally {
        reader.close();
      }
View Full Code Here

    writeSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile,
       srcFileRoot, testContainerId);

    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    try {
      LogReader.readAcontainerLogs(dis, writer);
    } catch (Exception e) {
View Full Code Here

    final int ch = filler;

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

    LogKey logKey = new LogKey(testContainerId);
    LogValue logValue =
        spy(new LogValue(Collections.singletonList(srcFileRoot.toString()),
            testContainerId, ugi.getShortUserName()));

    final CountDownLatch latch = new CountDownLatch(1);
View Full Code Here

    writeSrcFile(srcFilePath, "stdout", numChars);

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

    LogKey logKey = new LogKey(testContainerId);
    LogValue logValue =
        new LogValue(Collections.singletonList(srcFileRoot.toString()),
            testContainerId, ugi.getShortUserName());

    logWriter.append(logKey, logValue);
    logWriter.close();

    // make sure permission are correct on the file
    FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
    Assert.assertEquals("permissions on log aggregation file are wrong"
      FsPermission.createImmutable((short) 0640), fsStatus.getPermission());

    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    LogReader.readAcontainerLogs(dis, writer);
   
    String s = writer.toString();
View Full Code Here

    UserGroupInformation ugi =
        UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

    LogKey logKey = new LogKey(testContainerId1);
    String randomUser = "randomUser";
    LogValue logValue =
        spy(new LogValue(Collections.singletonList(srcFileRoot.toString()),
            testContainerId1, randomUser));
   
View Full Code Here

    try {
      Map<String, Map<String, String>> logMap =
          new HashMap<String, Map<String, String>>();
      DataInputStream valueStream;

      LogKey key = new LogKey();
      valueStream = reader.next(key);

      while (valueStream != null) {
        LOG.info("Found container " + key.toString());
        Map<String, String> perContainerMap = new HashMap<String, String>();
        logMap.put(key.toString(), perContainerMap);

        while (true) {
          try {
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintStream ps = new PrintStream(baos);
            LogReader.readAContainerLogsForALogType(valueStream, ps);

            String writtenLines[] = baos.toString().split(
              System.getProperty("line.separator"));

            Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
            String fileType = writtenLines[0].substring(9);

            Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
            String fileLengthStr = writtenLines[1].substring(11);
            long fileLength = Long.parseLong(fileLengthStr);

            Assert.assertEquals("Log Contents:",
              writtenLines[2].substring(0, 13));

            String logContents = StringUtils.join(
              Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
            perContainerMap.put(fileType, logContents);

            LOG.info("LogType:" + fileType);
            LOG.info("LogType:" + fileLength);
            LOG.info("Log Contents:\n" + perContainerMap.get(fileType));
          } catch (EOFException eof) {
            break;
          }
        }

        // Next container
        key = new LogKey();
        valueStream = reader.next(key);
      }

      // 1 for each container
      Assert.assertEquals(expectedContainerIds.length, logMap.size());
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.