/**
* Copyright © 2012 Alcatel-Lucent.
*
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
* Licensed to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.alu.e3.logger;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.channels.FileChannel;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang.StringEscapeUtils;
import com.alu.e3.Utilities;
import com.alu.e3.common.E3Constant;
import com.alu.e3.common.logging.Category;
import com.alu.e3.common.logging.CategoryLogger;
import com.alu.e3.common.logging.CategoryLoggerFactory;
import com.alu.e3.common.logging.LoggingUtil;
import com.alu.e3.common.logging.LoggingUtil.LogFileSource;
import com.alu.e3.common.logging.NonJavaLogger;
import com.alu.e3.common.tools.CommonTools;
import com.alu.e3.data.model.Instance;
import com.alu.e3.installer.NonExistingManagerException;
import com.alu.e3.installer.command.SSHCommand;
import com.alu.e3.installer.command.ShellCommandResult;
import com.alu.e3.osgi.api.ITopology;
import com.jcraft.jsch.JSchException;
/**
* The LogCollector class provides an object to traverse the topology
* and visit each instance, collecting log files to a repository on
* the system manager.
*
* There are still some unresolved issues in the design and/or
* implementation of the Log Collector. These include:
* <ul>
* <li> Throughout the code, instances are referred to by IP-Address.
* This is probably not right, and will have to be changed to some
* sort of (non-volatile) instance identifier.
* <li> If logging config files are moved or significantly edited
* (or not set up properly on install) the Collector may not be
* able to find remote log files.
* </ul>
*/
public class LogCollector implements Callable<Long> {
private static final String COLLECTION_PATH = System.getProperty("user.home") + "/apache-servicemix/data/logCollection";
private static final CategoryLogger logger = CategoryLoggerFactory.getLogger(LogCollector.class, Category.LOG);
// Should collected files be deleted on the source after collection?
private static final boolean deleteAfterCollect = true;
// If a file already exists with the copied log-file name, append a ".1" etc to uniquify?
// Don't uniquify if we don't delete source logs, since this will lead to local duplicates
private static final boolean uniquifyCopiedFilenames = deleteAfterCollect;
private static final int uniquifyLimit = 100; // unique filename extension limit, after this overwrite
// Should the collector only collect (and delete) files owned by the ssh user?
// This feature hasn't yet been tested ...
private static final boolean collectOnlyUserOwnedLogs = false;
// Set a short ssh connect timeout since we assume all instances are close,
// and on timeout we can try again on next collection
private static final int sshSessionTimeout = 10000; // 10s
/*
* Synchronization policy: allow only one writer to operate at a time.
* Allow concurrent readers and writer, except that readers must ignore
* any files the writer is currently writing, indicated by a specific suffix
*/
// For now: use a lock to allow only one writer (collector) to run at a time
private static final Lock writerLock = new ReentrantLock();
// Reader should ignore these files - collector is currently writing
private static final String workingFileSuffix = "~";
// Each logCollector instance gets its own serial number
// A class-static atomic variable keeps the serial numbers unique
private static final AtomicLong serialNumber = new AtomicLong();
private static long generateSerialNumber() { return serialNumber.getAndIncrement(); }
private static final AtomicLong lastCompletedCollector = new AtomicLong();
private final long collectorSerialNumber;
// Each collector can have its own topology (we could split large
// topologies for collection by multiple collectors)
private final ITopology topology;
/**
* Data structure to hold information about a (remote) file
*
*/
private static class FileInfo {
public String filePath;
public String fileSize;
public String fileModtime;
FileInfo(String path, String size, String modtime) {
filePath = path;
fileSize = size;
fileModtime = modtime;
}
}
/**
* File-name filter to check for log files (by extension)
* and to ignore "working files" (files being written by collector)
*/
private static class LogFileFilter implements FileFilter
{
private String basename;
public LogFileFilter(String basename)
{
this.basename = basename;
}
public boolean accept(File pathname)
{
String filename = pathname.getName();
if (basename != null) {
return ((filename.matches(basename + LoggingUtil.rollingFileExtPatternString) ||
filename.matches(basename + LoggingUtil.dailyRollingFileExtPatternString) &&
!filename.endsWith(LogCollector.workingFileSuffix)));
} else {
return ((filename.matches(LoggingUtil.rollingFilePatternString) ||
filename.matches(LoggingUtil.dailyRollingFilePatternString) &&
!filename.endsWith(LogCollector.workingFileSuffix)));
}
}
}
/**
* LogCollector constructor without parameters to satisfy spring-bean requirements.
*/
public LogCollector(ITopology topology)
{
this.topology = topology;
this.collectorSerialNumber = LogCollector.generateSerialNumber();
}
public long getCollectorID()
{
return collectorSerialNumber;
}
public static long getLastCollectorID()
{
return lastCompletedCollector.longValue();
}
/**
* Traverse the topology and collect logs from each instance found.
*
* @return The number of bytes collected in all logs
* @throws InterruptedException
* @throws TimeoutException
*/
@Override
public Long call() throws InterruptedException, TimeoutException, NonExistingManagerException
{
if(logger.isDebugEnabled()) {
logger.debug("Call to LogCollector.call()");
}
long bytesCollected = collectAllLogs(0, TimeUnit.SECONDS);
return Long.valueOf(bytesCollected);
}
/**
* Traverse the topology and collect logs from each instance found.
*
* @param waitTimeout The time to wait if another collector is running
* @param unit The time unit for waitTimeout
* @return The number of bytes collected in all logs
* @throws InterruptedException
* @throws TimeoutException
* @throws NonExistingManagerException
*/
public long collectAllLogs(long waitTimeout, TimeUnit unit) throws InterruptedException, TimeoutException, NonExistingManagerException
{
DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd 'at' HH:mm:ss z");
String launchDate = dateFormat.format(new Date());
logger.debug("Launching system-manager log-file collection ({}) ...", launchDate);
if (!LogCollector.writerLock.tryLock(waitTimeout, unit)) {
logger.warn("Attempt to run log collector but cannot acquire lock (another collection must be running)");
throw new TimeoutException("Timeout waiting to acquire log collector write lock");
}
long collectionCount = 0L;
logger.debug("LogCollector ID: {}", getCollectorID());
try {
Set<String> visitedIPs = new HashSet<String>();
// Create the top-level log collection directory, if this is the first time run
File collectionDir = new File(COLLECTION_PATH);
if (!collectionDir.exists() && !collectionDir.mkdirs()) {
logger.error("Unable to create log-collection directory: {}", COLLECTION_PATH);
return 0L;
}
// Iterate through all instances in the current topology
Instance logCollectorInstance = Utilities.getManagerByIP(CommonTools.getLocalHostname(), CommonTools.getLocalAddress(), topology.getInstancesByType(E3Constant.E3MANAGER), logger);
List<Instance> instances = getInstanceList(this.topology);
if(logger.isDebugEnabled()) {
logger.debug("There are {} instances in the current topology", instances.size());
}
for (Instance logSource : instances) {
LogCollector.logInstance(logSource); // for debugging
// Avoid visiting the same address twice
String ipAddress = logSource.getInternalIP();
if (ipAddress == null) {
logger.warn("Encountered instance node with null ipAddress during log collection!");
continue;
}
if (CommonTools.isLocal(ipAddress)) {
ipAddress = E3Constant.localhost; // stay consistent
}
if (visitedIPs.contains(ipAddress)) {
if(logger.isDebugEnabled()) {
logger.debug("Skipping already-visited address: {}", ipAddress);
}
continue;
}
visitedIPs.add(ipAddress);
// Create or verify the existence of a log-collection target directory
String sanitizedHost = ipToCollectionDirectory(ipAddress);
File instanceCollectionDir = new File(COLLECTION_PATH, sanitizedHost);
if (instanceCollectionDir.exists()) {
if (!instanceCollectionDir.isDirectory()) {
logger.error("Log-collection target exists but is not a directory: {}", instanceCollectionDir.getAbsolutePath());
continue;
}
} else {
if (!instanceCollectionDir.mkdirs()) {
logger.error("Unable to create log-collection directory: {}", instanceCollectionDir.getAbsolutePath());
continue;
}
}
// Finally, perform log collection
// There may be a chance for parallelism here by farming the collection work for each instance
// out to a separate worker thread. At a minimum the local collection could occur in parallel with
// collection on a remote host.
if (ipAddress.equalsIgnoreCase(E3Constant.localhost)) {
try {
collectionCount += collectAllLocalLogs(instanceCollectionDir);
} catch (IOException ex) {
logger.warn("Error trying to copy local log files to {}", instanceCollectionDir.getAbsolutePath());
}
} else {
try {
collectionCount += collectAllRemoteLogs(logSource, logCollectorInstance, instanceCollectionDir);
} catch (JSchException ex) {
if(logger.isDebugEnabled()) {
logger.debug("Could not connect to host: {}", logSource.getInternalIP());
logger.debug(ex.getLocalizedMessage());
}
} catch (IOException ex) {
if(logger.isDebugEnabled()) {
logger.debug("Got IOException while connecting to or transferring files from host: {}", logSource.getInternalIP());
logger.debug(ex.getLocalizedMessage());
}
}
}
// At this point the collection has "completed", even if IOExceptions could have
// occurred and been caught above
LogCollector.lastCompletedCollector.set(getCollectorID());
logger.debug("Completed log collection with ID: {} ({})", getCollectorID(), dateFormat.format(new Date()));
}
} finally {
LogCollector.writerLock.unlock();
}
return collectionCount;
}
/**
* Traverse the topology and collect a certain number of lines from the
* active logs on each instance found.
*
* @param numLines The number of lines to retrieve from each log file on each instance
* @return An XML-structured string with a single <code><logCollection></code> node and
* <code><log></code> node for each log-file found (including multiple instances).
* @throws InterruptedException
* @throws TimeoutException
*/
public String collectAllActiveLogs(int numLines)
{
StringBuilder sb = new StringBuilder();
Set<String> visitedIPs = new HashSet<String>();
// Iterate through all instances in the current topology
// (Note: use TRACE-level log statements so that our logging doesn't appear in returned log tails)
List<Instance> instances = getInstanceList(this.topology);
logger.trace("There are {} instances in the current topology", instances.size());
Instance collectorInstance;
try {
collectorInstance = Utilities.getManagerByIP(CommonTools.getLocalHostname(), CommonTools.getLocalAddress(), topology.getInstancesByType(E3Constant.E3MANAGER), logger);
} catch (NonExistingManagerException e) {
String errMsg = "Cannot find manager at " + CommonTools.getLocalAddress() + " while collecting active logs.";
logger.error(errMsg);
return errMsg;
}
for (Instance instance : instances) {
// Avoid visiting the same address twice
String ipAddress = instance.getInternalIP();
if (ipAddress == null) {
logger.warn("Encountered instance node with null ipAddress during log collection!");
continue;
}
if (CommonTools.isLocal(ipAddress)) {
ipAddress = E3Constant.localhost; // stay consistent
}
if (visitedIPs.contains(ipAddress)) {
logger.trace("Skipping already-visited address: {}", ipAddress);
continue;
}
visitedIPs.add(ipAddress);
// Finally, perform log collection (of active-log tails)
if (ipAddress.equalsIgnoreCase(E3Constant.localhost)) {
try {
String logs = getTailOfLocalActiveLogs(numLines);
if (logs != null) {
sb.append(logs);
}
} catch (IOException ex) {
logger.warn("Error trying to get tail of active log files from {}", ipAddress);
}
} else {
try {
String logs = getTailOfRemoteActiveLogs(instance, collectorInstance, numLines);
if (logs != null) {
sb.append(logs);
}
} catch (JSchException ex) {
logger.warn("Could not connect to host: {}", instance.getInternalIP());
logger.warn(ex.getLocalizedMessage());
} catch (IOException ex) {
logger.warn("Got IOException while connecting to or transferring files from host: {}", instance.getInternalIP());
logger.warn(ex.getLocalizedMessage());
}
}
}
return LoggingResponseBuilder.logCollectionToXml(sb.toString());
}
/**
* A top-level method to return a certain number of most-recent log
* lines from previously-connected log files from all instances.
*
* @param numLines The requested number of log lines to retrieve.
* Fewer lines may be returned if sufficient log entries are not available.
* @return An XML-structured string with a single <code><logCollection></code> node and
* <code><log></code> node for each log-file found (including multiple instances).
*/
public String getCollectedLogLines(int numLines)
{
File collectionDir = new File(COLLECTION_PATH);
if (!collectionDir.exists() || !collectionDir.isDirectory()) {
return null;
}
if(logger.isDebugEnabled()) {
logger.debug("Attempting to get all collected logs ...");
}
StringBuilder logLines = new StringBuilder();
File[] files = collectionDir.listFiles(); // get a list of instance directories
for (File item : files) {
if(logger.isDebugEnabled()) {
logger.debug("Collection directory entry: {}", item.getAbsolutePath());
}
if (item.isDirectory()) {
String ipAddress = collectionDirectoryToIP(item.getName());
if(logger.isDebugEnabled()) {
logger.debug("Getting logs from: {}", ipAddress);
}
// First get the java logs
String contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.JAVA, numLines);
if (contents != null) {
logLines.append(contents);
}
// Next get the servicemix logs
contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.SMX, numLines);
if (contents != null) {
logLines.append(contents);
}
// Get the E3-facility syslog files
contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.SYSLOG, numLines);
if (contents != null) {
logLines.append(contents);
}
}
}
return LoggingResponseBuilder.logCollectionToXml(logLines.toString());
}
/**
* Returns the specified number of most-recent log lines from previously-
* collected log files for a particular instance and log source.
*
* @param ipAddress The ip-address of the instance
* @param logSource The source for the logs (JAVA, SMX, SYSLOG)
* @param numLines The number of lines to retrieve. Fewer lines may be returned if the
* requested number is not available.
* @return Log lines in XML structure, with a top-level <code><log></code> node.
*/
public String getCollectedLogLinesFromInstance(String ipAddress, LogFileSource logSource, int numLines)
{
StringBuilder logLines = null;
String logFilePath = null;
int lineCount = 0;
File collectionDir = new File(COLLECTION_PATH, LogCollector.ipToCollectionDirectory(ipAddress));
if (!collectionDir.exists() || !collectionDir.isDirectory()) {
logger.warn("No log collection directory for ipAddress: {}", ipAddress);
return null;
}
File logSourceSubdir = new File(collectionDir, logSource.toString());
if (!logSourceSubdir.exists() || !logSourceSubdir.isDirectory()) {
logger.warn("No log-type '{}' collection subdirectory for ipAddress: {}", logSource.toString(), ipAddress);
return null;
}
// Get the list of collected log files in date order
File[] files = logSourceSubdir.listFiles(new LogFileFilter(null)); // get all log files, regardless of basename and ext
Arrays.sort(files, new Comparator<Object>() {
public int compare(Object o1, Object o2) {
// Sort by decreasing date first, and then decreasing alphabetical
File f1 = (File)o1; File f2 = (File)o2;
int result = (Long.valueOf(f2.lastModified())).compareTo(Long.valueOf(f1.lastModified()));
if (result == 0) {
result = f2.getName().compareTo(f1.getName());
}
return result;
}
});
logger.trace("Sorted log-files:");
for (File logFile : files) {
logger.trace("{}", logFile.getName());
}
logLines = new StringBuilder();
try {
for (File file : files) {
String fileName = file.getName();
if(logger.isDebugEnabled()) {
logger.debug("Consider file: {}", fileName);
}
logFilePath = file.getAbsolutePath();
if(logger.isDebugEnabled()) {
logger.debug("Retrieving {} log lines from file {}", String.valueOf(numLines-lineCount), logFilePath);
}
String logContent = LogCollector.getTailOfFile(file, numLines - lineCount);
logLines.insert(0, logContent);
int retrievedCount = lineCount(logContent);
if(logger.isDebugEnabled()) {
logger.debug("Actually got {} lines", String.valueOf(retrievedCount));
}
lineCount += retrievedCount;
if (lineCount >= numLines) {
break;
}
}
} catch (IOException ex) {
// Swallow exception from any one file read and hope to get lines from the next log
logger.warn("Couldn't read from log file {}", logFilePath == null ? "(null)" : logFilePath);
}
if(logger.isDebugEnabled()) {
logger.debug("Got {} of {} requested lines", String.valueOf(lineCount), String.valueOf(numLines));
}
return LoggingResponseBuilder.logLinesToXml(logSource, ipAddress,
StringEscapeUtils.escapeXml(logLines.toString()));
}
/**
* Collects logs from the localhost for all log sources.
*
* @param instanceCollectionDir Target directory to place collected logs (subdirectories for
* each log source will be created)
* @return The number of bytes in all files collected
* </ul>
* @throws IOException
*/
private long collectAllLocalLogs(File instanceCollectionDir) throws IOException
{
// First, get the E3Appender (java) logs
// Parse the log-config file to find path to logs
long bytesCollected = 0L;
String logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.JAVA);
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// If we can't determine the log-file path from the config file,
// look anyway in the usual servicemix log directory for any log files with the default name
logger.warn("Localhost is not using E3Appender, using default log-file path (check log-config file: {})", LoggingUtil.defaultConfigPath);
logFilePath = LoggingUtil.defaultLogPath;
}
if(logger.isDebugEnabled()) {
logger.debug("java log file path {}", logFilePath);
}
File logFile = new File(logFilePath);
bytesCollected += collectLocalLogs(logFile.getParentFile(), instanceCollectionDir, logFile.getName(), LogFileSource.JAVA);
// Next, get the serviceMix logs
logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.SMX);
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// Same situation as above
logger.warn("Localhost log-config file ({}) does not specify servicemix log location, using default", LoggingUtil.defaultConfigPath);
logFilePath = LoggingUtil.defaultSMXLogPath;
}
if(logger.isDebugEnabled()) {
logger.debug("smx log file path {}", logFilePath);
}
File smxLogFile = new File(logFilePath);
bytesCollected += collectLocalLogs(smxLogFile.getParentFile(), instanceCollectionDir, smxLogFile.getName(), LogFileSource.SMX);
// Collect the E3-specific syslog files
logFilePath = NonJavaLogger.getLogFilePath();
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// Same situation as above
logFilePath = NonJavaLogger.defaultLogFilePath;
logger.warn("Localhost syslog-config file ({}) does not specify log location, using default", logFilePath);
}
if(logger.isDebugEnabled()) {
logger.debug("syslog file path: {}", logFilePath);
}
File syslogFile = new File(logFilePath);
bytesCollected += collectLocalLogs(syslogFile.getParentFile(), instanceCollectionDir, syslogFile.getName(), LogFileSource.SYSLOG);
return bytesCollected;
}
/**
* Collects logs from the specified instance for all log sources.
*
* @param logSource The topology instance to collect logs from
* @param instanceCollectionDir Target directory to place collected logs (subdirectories for
* each log source will be created)
* @return The number of bytes in all files collected
* @throws JSchException, IOException
*/
private long collectAllRemoteLogs(Instance logSource, Instance logDestination, File instanceCollectionDir) throws JSchException, IOException
{
if (logSource == null) {
throw new NullPointerException("The instance than contains remote logs cannot be null");
}
if (logDestination == null) {
throw new NullPointerException("The instance where to copy logs cannot be null");
}
// First, try to open a new SSH session to instance
long bytesCollected = 0L;
String ipAddress = logSource.getInternalIP();
if(logger.isDebugEnabled()) {
logger.debug("trying to connect to {} via ssh ...", ipAddress);
}
SSHCommand sshCommand = new SSHCommand();
sshCommand.connect(logDestination.getSSHKey(), ipAddress, 22, logSource.getUser(), logSource.getPassword(), sshSessionTimeout);
// Start with E3Appender Java logs first
// Get a local copy of the logging config file to determine log-file path
String remoteLogPath = null;
File localConfigFile = new File(instanceCollectionDir, "java-logging.cfg");
String localConfigFilePath = localConfigFile.getAbsolutePath();
if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.JAVA)) {
remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.JAVA, false);
} else {
logger.warn("Couldn't retrieve E3 Java logging config file from host {}, will try default path", ipAddress);
}
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// If we can't find a logging config file with an E3Appender section,
// look anyway in the usual servicemix log directory for any log files with the default name
logger.warn("Instance at {} is not using E3Appender (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath);
remoteLogPath = LoggingUtil.defaultLogPath;
}
File localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.JAVA);
if (localTargetDir == null) {
logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.JAVA.toString());
} else {
File remoteLog = new File(remoteLogPath);
List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName());
for (FileInfo remoteFileInfo : logList) {
File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir));
try {
bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect);
} catch (Exception ex) {
// Continue copy attempts if we experience an error
logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage());
}
}
}
// Now try to get the remote serviceMix log files
// We use the same log-config file as the java logs to parse the path
remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.SMX, false);
localConfigFile.delete(); // no longer needed
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// If we can't find a logging config file with the proper appender section,
// look anyway in the usual servicemix log directory for any log files with the default name
logger.warn("Instance at {} is not using expected appender for servicemix rootLogger (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath);
remoteLogPath = LoggingUtil.defaultSMXLogPath;
}
localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.SMX);
if (localTargetDir == null) {
logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.SMX.toString());
} else {
File remoteLog = new File(remoteLogPath);
List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName());
for (FileInfo remoteFileInfo : logList) {
File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir));
try {
bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect);
} catch (Exception ex) {
// Continue copy attempts if we experience an error
logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage());
}
}
}
// Collect the E3-specific syslog files
// For syslog we parse the rsyslog config file for the log-file path
localConfigFile = new File(instanceCollectionDir, "syslog.cfg");
localConfigFilePath = localConfigFile.getAbsolutePath();
if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.SYSLOG)) {
remoteLogPath = NonJavaLogger.getLogFilePathFromConfigFile(localConfigFilePath);
localConfigFile.delete();
} else {
logger.warn("Couldn't retrieve E3 syslog config file from host {}", ipAddress);
}
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// Try default path
remoteLogPath = NonJavaLogger.defaultLogFilePath;
logger.warn("Instance at {} does not specify an E3-specific syslog file, trying default: {}", ipAddress, remoteLogPath);
}
localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.SYSLOG);
if (localTargetDir == null) {
logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.SYSLOG.toString());
} else {
File remoteLog = new File(remoteLogPath);
List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName());
for (FileInfo remoteFileInfo : logList) {
File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir));
try {
bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect);
} catch (Exception ex) {
// Continue copy attempts if we experience an error
logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage());
}
}
}
// We're done - disconnect and return number of bytes copied
sshCommand.disconnect();
if(logger.isDebugEnabled()) {
logger.debug("connected/disconnected!");
}
return bytesCollected;
}
/**
* Visit a particular local directory and collect all the log files that start with
* a particular base filename, copying them to the specified target directory.
*
* @param sourceDir The directory in which the log files are located
* @param targetDir The directory to put the copied files
* @param baseName The basename of the log files to collect (such as "e3.log", "servicemix.log", etc)
* @param logSource The type of logs to collect (JAVA, SMX, SYSLOG); the string form of the type will
* be used as a destination subdirectory under targetDir
* @return The number of bytes in all files collected
*/
private long collectLocalLogs(File sourceDir, File targetDir, final String baseName, LogFileSource logSource)
{
long bytesCollected = 0L;
logger.debug("Collecting logs from localhost from {} with base {}", sourceDir, baseName);
// New: Get all log files with a matching basename, regardless of rotation type
File[] logFiles = sourceDir.listFiles(new LogFileFilter(baseName));
if (logFiles == null) {
logger.warn("Error retrieving file list from {} matching name {}", sourceDir, baseName);
return 0L;
}
// Make or use a specific subdirectory for this log type
targetDir = new File(targetDir, logSource.toString());
if (!targetDir.exists()) {
targetDir.mkdirs();
} else if (!targetDir.isDirectory()) {
logger.error("Target for local log collection is not a directory: {}", targetDir.getAbsolutePath());
return 0L;
}
File logFile;
for (File log : logFiles) {
logFile = log;
if(logger.isDebugEnabled()) {
logger.debug("Copying log file {} ...", logFile.getAbsolutePath());
}
String destFileName = targetNameForLogFile(logFile, targetDir);
File destFile = new File(targetDir, destFileName);
try {
bytesCollected += LogCollector.copyLocalFileWithWorkingTemp(logFile, destFile, deleteAfterCollect);
} catch (IOException ex) {
// Continue copying despite single-file error
logger.error("Could not copy file {}: {}", logFile.getAbsolutePath() + " to " + targetDir.getAbsolutePath(), ex.getLocalizedMessage());
}
}
return bytesCollected;
}
/**
* Collects logs from the localhost for all log sources.
*
* @param numLines The requested number of log lines to retrieve.
* Fewer lines may be returned if sufficient log entries are not available.
* @return An XML-structured string with a <code><log></code> node for
* each log-file found..
* @throws IOException
*/
private String getTailOfLocalActiveLogs(int numLines) throws IOException
{
// First, get the E3Appender (java) log
// Parse the log-config file to find path to log file
// Note: use TRACE-level logging here since our output may appear in retrieved log lines
StringBuilder sb = new StringBuilder();
String logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.JAVA);
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// If we can't determine the log-file path from the config file,
// look anyway in the usual servicemix log directory for any log files with the default name
logger.warn("Localhost is not using E3Appender, using default log-file path (check log-config file: {})", LoggingUtil.defaultConfigPath);
logFilePath = LoggingUtil.defaultLogPath;
}
logger.trace("java log file path {}", logFilePath);
File logFile = new File(logFilePath);
String logLines = execTailOnFile(logFile, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString())));
}
// Next, get the serviceMix log
logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.SMX);
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// Same situation as above
logger.warn("Localhost log-config file ({}) does not specify servicemix log location, using default", LoggingUtil.defaultConfigPath);
logFilePath = LoggingUtil.defaultSMXLogPath;
}
logger.trace("local smx log file path {}", logFilePath);
File smxLogFile = new File(logFilePath);
logLines = execTailOnFile(smxLogFile, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SMX, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString())));
}
// Collect the E3-specific syslog files
logFilePath = NonJavaLogger.getLogFilePath();
if ((logFilePath == null) || (logFilePath.length() == 0)) {
// Same situation as above
logFilePath = NonJavaLogger.defaultLogFilePath;
logger.warn("Localhost syslog-config file does not specify an E3-specific log location, using default: {}", LoggingUtil.defaultLogPath);
}
logger.trace("local syslog file path: {}", logFilePath);
logFile = new File(logFilePath);
logLines = execTailOnFile(logFile, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SYSLOG, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString())));
}
return sb.toString();
}
/**
* Retrieves the last <code>numLines</code> lines from the remote machine's active logs.
*
* @param numLines The requested number of log lines to retrieve.
* Fewer lines may be returned if sufficient log entries are not available.
* @return An XML-structured string with a <code><log></code> node for
* each log-file found..
* @throws JSchException, IOException
*/
private String getTailOfRemoteActiveLogs(Instance logSource, Instance logDestination, int numLines) throws JSchException, IOException
{
if (logSource == null) {
throw new NullPointerException ("Log source cannot be null");
}
if (logDestination == null) {
throw new NullPointerException ("Log destination cannot be null");
}
StringBuilder sb = new StringBuilder();
// First, try to open a new SSH session to instance
// Note: use TRACE-level logging here since our output may appear in retrieved log lines
String ipAddress = logSource.getInternalIP();
logger.trace("trying to connect to {} via ssh ...", ipAddress);
SSHCommand sshCommand = new SSHCommand();
sshCommand.connect(logDestination.getSSHKey(), ipAddress, 22, logSource.getUser(), logSource.getPassword(), sshSessionTimeout);
// Start with E3Appender Java log first
// Get a local copy of the logging config file to determine log-file path
String remoteLogPath = null;
File localConfigFile = File.createTempFile("java-logging", ".cfg");
boolean gotConfigFile = false;
String localConfigFilePath = localConfigFile.getAbsolutePath();
if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.JAVA)) {
gotConfigFile = true;
remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.JAVA, false);
} else {
logger.warn("Couldn't retrieve E3 Java logging config file from host {}, will try default path", ipAddress);
}
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// If we can't find a logging config file with an E3Appender section,
// look anyway in the usual servicemix log directory for any log files with the default name
logger.warn("Instance at {} is not using E3Appender (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath);
remoteLogPath = LoggingUtil.defaultLogPath;
}
logger.trace("java log file path for instance {}: {}", ipAddress, remoteLogPath);
String logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, ipAddress, StringEscapeUtils.escapeXml(logLines.toString())));
}
// Next, get the serviceMix log
// We use the same logging config file to get the servicemix log-file path
remoteLogPath = null;
if (gotConfigFile) {
remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.SMX, false);
localConfigFile.delete(); // We're done with the local version of the java/smx config file
}
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// Same situation as above
logger.warn("Could not parse servicemix log-file path from config file for instance at {}, using default", ipAddress);
remoteLogPath = LoggingUtil.defaultSMXLogPath;
}
logger.trace("smx log file path for instance {}: {}", ipAddress, remoteLogPath);
logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SMX, ipAddress, StringEscapeUtils.escapeXml(logLines.toString())));
}
// Collect the E3-specific syslog files
// For syslog we parse the rsyslog config file for the log-file path
remoteLogPath = null;
localConfigFile = File.createTempFile("syslog", ".cfg");
localConfigFilePath = localConfigFile.getAbsolutePath();
if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.SYSLOG)) {
remoteLogPath = NonJavaLogger.getLogFilePathFromConfigFile(localConfigFilePath);
localConfigFile.delete();
} else {
logger.warn("Couldn't retrieve E3 syslog config file from host {}", ipAddress);
}
if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) {
// Same situation as above
logger.warn("Could not parse E3-specific log-file path from syslog config file for instance at {}, using default", ipAddress);
remoteLogPath = NonJavaLogger.defaultLogFilePath;
}
logger.trace("syslog file path: {}", remoteLogPath);
logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines);
if (logLines != null) {
sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SYSLOG, ipAddress, StringEscapeUtils.escapeXml(logLines.toString())));
}
return sb.toString();
}
// Code based on Stack Overflow suggestion:
// http://stackoverflow.com/questions/686231/java-quickly-read-the-last-line-of-a-text-file
// Pretty basic: byte-based (no Unicode) and relies on Unix-style EOL 0xA
private static String getTailOfFile(File file, int numLines) throws FileNotFoundException, IOException
{
if (numLines < 0) {
return null;
} else if (numLines == 0) {
return "";
}
java.io.RandomAccessFile raFile = new java.io.RandomAccessFile(file, "r");
long fileLength = file.length() - 1;
StringBuilder sb = new StringBuilder();
int line = 0;
for (long filePointer = fileLength; filePointer >= 0; filePointer--) {
raFile.seek(filePointer);
int readByte = raFile.readByte();
if (readByte == 0xA) {
if (filePointer < fileLength) {
line = line + 1;
if (line >= numLines) {
break;
}
}
}
sb.append((char)readByte);
}
String lastLines = sb.reverse().toString();
return lastLines;
}
private static String execTailOnFile(File file, int numLines) throws IOException
{
String numLinesArg = String.valueOf(numLines);
if ((numLines < 0) || (numLinesArg == null) || (numLinesArg.length() == 0) || (file == null) || !file.exists()) {
return null;
} else if (numLines == 0) {
return "";
}
ProcessBuilder processBuilder = new ProcessBuilder("/usr/bin/tail", "-n " + numLinesArg, file.getAbsolutePath());
File workingDirectory = file.getParentFile();
if (workingDirectory != null) {
processBuilder.directory(workingDirectory);
}
Process p = processBuilder.start();
// Get tail's output: its InputStream
InputStream is = p.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader reader = new BufferedReader(isr);
StringBuilder sb = new StringBuilder();
String line;
if (reader != null) {
try {
while ((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
} finally {
try {
if (reader != null) {
reader.close();
}
if (is != null) {
is.close();
}
} catch (IOException ioe) {
// Nothing to do on close exception
}
}
}
/*
// We only need to wait for p to finish if we want the exit value
try {
p.waitFor();
} catch (InterruptedException ex) {
logger.warn("Tail on logfile {} interrupted!", file.getAbsoluteFile());
}
logger.debug("Tail process exited with code {} ", String.valueOf(p.exitValue()));
*/
return sb.toString();
}
/**
* Count the number of newline characters in a string.
*
* @param logLines The string to parse (assumed to be log lines)
* @return The number of newlines found
*/
private static int lineCount(String logLines)
{
return logLines.split(System.getProperty("line.separator")).length;
}
/**
* Traverse the specified topology and return a list of instances.
* Included instance types:
* <ul>
* <li>E3Gateway
* <li>E3GatewayA
* <li>E3Manager
* <li>E3ManagerA
* </ul>
*
* @param t The topology to traverse
* @return A List of instances found in the topology
*/
private List<Instance> getInstanceList(ITopology t)
{
List<Instance> instances = new LinkedList<Instance>();
if (t == null) {
logger.warn("topology is null when trying to retrieve instances!");
} else {
instances.addAll(t.getInstancesByType("E3Gateway"));
instances.addAll(t.getInstancesByType("E3Manager"));
instances.addAll(t.getInstancesByType("E3GatewayA"));
instances.addAll(t.getInstancesByType("E3ManagerA"));
// add other types?
}
return instances;
}
/*
* Local file operations
*/
/**
* Copy a file from one location to another on the localhost, first moving
* (renaming) the source file out of the way of any rotator process, and
* then optionally deleting the source file after a successful copy.
* Will attempt to replicate the modification time from the original file.
*
* @param sourceFile File to copy
* @param destFile Destination file
* @param deleteSource If <code>true</code>, will delete original after copy
* @return The number of bytes copied
* @throws IOException
*/
public static long copyLocalFile(File sourceFile, File destFile, boolean deleteSource) throws IOException
{
long bytesCopied = 0L;
if ((sourceFile == null) || (destFile == null)) {
throw new NullPointerException("Source or destination file is null (source: " + sourceFile + ", dest: " + destFile + ")");
}
if (!destFile.exists()) {
destFile.createNewFile();
}
String origSourcePath = sourceFile.getPath();
File tempFile = new File(tempNameForSourceFile(sourceFile.getPath()));
FileChannel source = null;
FileChannel destination = null;
IOException cleanupException = null;
boolean success = false;
// Copy and validate result
try {
// Rename source file to temporary name before copying
if(logger.isDebugEnabled()) {
logger.debug("Renaming local file to: {}", tempFile.getPath());
}
if (!sourceFile.renameTo(tempFile)) {
logger.error("Could not move file to new name: {}", tempFile.getAbsolutePath());
} else {
source = new FileInputStream(tempFile).getChannel();
destination = new FileOutputStream(destFile).getChannel();
bytesCopied = destination.transferFrom(source, 0, source.size());
copyModificationTime(tempFile, destFile);
// Check integrity of copy
success = validateFileCopy(tempFile, destFile);
if (!success) {
logger.warn("Copy of file {} did not pass integrity check!", origSourcePath);
}
}
} catch (IOException ex) {
// If there's been an error copying the file, we may be left with a zero-length or incomplete file
if (!success) {
if(logger.isDebugEnabled()) {
logger.debug("Deleting failed copy of local file: {}", destFile.getAbsolutePath());
}
destFile.delete();
}
} finally {
// Use a try-block during cleanup, but only throw exception if the
// main file-copy try-block doesn't
try {
if (source != null) {
source.close();
}
if (destination != null) {
destination.close();
}
if (deleteSource && success) {
if(logger.isDebugEnabled()) {
logger.debug("Deleting local source file: {}", tempFile.getAbsolutePath());
}
tempFile.delete();
} else {
// Move source file back from temp name
if (tempFile == null || !tempFile.renameTo(new File(origSourcePath))) {
logger.error("Could not restore original filename: {}", origSourcePath);
}
}
} catch (IOException ex) {
logger.warn("IOException during local file-copy cleanup: {}", ex);
cleanupException = new IOException(ex);
}
}
if (cleanupException != null) {
throw cleanupException;
}
return bytesCopied;
}
/**
* Copy the modification time from one local file to another.
*
* @param sourceFile The file to copy the mod time from
* @param destFile The file to copy the mod time to
* @throws IOException
*/
public static void copyModificationTime(File sourceFile, File destFile) throws IOException
{
destFile.setLastModified(sourceFile.lastModified());
}
private static boolean validateFileCopy(File sourceFile, File destFile) throws IOException
{
boolean success;
long sourceLength = sourceFile.length();
long destLength = destFile.length();
if (sourceLength != destLength) {
logger.error("File-size difference after copy of file {}: {}",
sourceFile.getAbsolutePath(), "orig size: " + String.valueOf(sourceLength) + ", copy size: " + String.valueOf(destLength));
success = false;
} else {
// Check if last lines of files are the same
String sourceTail = execTailOnFile(sourceFile, 1);
String destTail = execTailOnFile(destFile, 1);
success = sourceTail != null && sourceTail.equals(destTail);
if (!success) {
logger.error("Last lines of copied files are different: '{}' vs '{}'", sourceTail, destTail);
}
}
return success;
}
/**
* Make a copy of a local file, but write first to a "working" temporary file and then
* rename the temporary file to the destination name.
*
* @param sourceFile The file to copy
* @param destFile The final destination of the copy
* @param deleteSource If <code>true</code>, will delete original after copy
* @return The number of bytes copied
* @throws IOException
*/
public static long copyLocalFileWithWorkingTemp(File sourceFile, File destFile, boolean deleteSource) throws IOException
{
File tempLocalFile = new File(tempNameForWorkingFile(destFile.getPath()));
long bytesCopied = copyLocalFile(sourceFile, tempLocalFile, deleteSource);
tempLocalFile.renameTo(destFile);
return bytesCopied;
}
/*
* Remote file operations
*/
/**
* Copies the appropriate logging-config file for the logSource from the ssh remote host to
* the path specified by destFilePath. May search remote host at multiple locations for
* config file, or use a default path.
*
* @param sshCommand An connected SSHCommand session
* @param destFilePath The target path for the copy of the config file
* @param logSource The type of log-config file to look for
* @return <ul>
* <li><code>true</code> if an appropriate log-config file was found and copied
* <li><code>false</code> otherwise
* </ul>
*/
private static boolean copyRemoteConfigFile(SSHCommand sshCommand, String destFilePath, LogFileSource logSource)
{
boolean success = false;
if (logSource.equals(LogFileSource.JAVA)) {
try {
// We assume there is only one location for the java-logging config file
String remoteConfigPath = LoggingUtil.defaultConfigPath;
success = (sshCommand.copyFrom(remoteConfigPath, destFilePath) > 0);
} catch (Exception ex) {
// swallow exception since failure means we try default paths, etc
}
} else if (logSource.equals(LogFileSource.SYSLOG)) {
// There are a couple of pre-defined alternatives for syslog config file paths
// And NonJavaLogger.getConfigFilePath() could return a non-default value if setConfigFilePath() has been used
String [] altPaths = NonJavaLogger.getAltConfigFilePaths();
List<String> pathCandidates = new LinkedList<String>();
pathCandidates.add(NonJavaLogger.getConfigFilePath());
pathCandidates.addAll(Arrays.asList(altPaths));
//boolean foundConfigFile = false;
for (String pathCand : pathCandidates) {
String remoteConfigPath = pathCand;
try {
if(logger.isDebugEnabled()) {
logger.debug("Trying syslog config path: {}", pathCand);
}
if (sshCommand.copyFrom(remoteConfigPath, destFilePath) > 0) {
success = true;
break;
}
} catch (Exception ex) {
if(logger.isDebugEnabled()) {
logger.debug("Failed on path {}", remoteConfigPath);
}
}
}
} else {
logger.warn("Invalid log-file type passed to getRemoteLogPath: {}", logSource.toString());
}
return success;
}
/**
* Creates a local directory to hold collected log files, specific to the log source (JAVA, SMX, SYSLOG)
*
* @param instanceCollectionDir The parent directory for the new directory
* @param logSource The log-file source (determines name of created directory)
* @return A File object representing the new directory, <code>null</code> on error
*/
private static File createLocalLogTargetDir(File instanceCollectionDir, LogFileSource logSource)
{
File logTargetDir = new File(instanceCollectionDir, logSource.toString());
if (logTargetDir.exists()) {
if (!logTargetDir.isDirectory()) {
logger.error("Log collection target path is not a directory: {}", logTargetDir.getAbsolutePath());
return null;
}
} else if (!logTargetDir.mkdirs()) {
logger.error("Could not create log collection target directory: {}", logTargetDir.getAbsolutePath());
}
return logTargetDir;
}
/**
* Searches the contents of the specified remote directory, and returns a list of FileInfo
* instances that represent remote files that match the specified basename.
*
* @param sshCommand An active (connected) SSHCommand
* @param remoteDirPath The path on the remote machine to search for matching files
* @param baseName A base filename to compare against each remote filename (no path) to determine a match
* @return A List of matching files in FileInfo form, empty if no matches were found
* @throws JSchException
* @throws IOException
*/
private static List<FileInfo> getMatchingRemoteFileList(SSHCommand sshCommand, String remoteDirPath, String baseName) throws JSchException, IOException
{
if ((sshCommand == null) || !sshCommand.isConnected()) {
throw new JSchException("Not connected with a valid SSH session");
}
String remoteUsername = null;
if (LogCollector.collectOnlyUserOwnedLogs) {
remoteUsername = sshCommand.getSessionUsername();
if (remoteUsername == null) {
throw new JSchException("Username for SSH session is not valid");
}
}
// First get a list of *all* files in the remote directory, and then
// parse the results for a match with the target basename
// (Prefer to do the matching work locally rather than remotely)
List<FileInfo> matches = new LinkedList<FileInfo>();
String findCmd;
if (LogCollector.collectOnlyUserOwnedLogs) {
findCmd = "find " + remoteDirPath + " -maxdepth 1 -user " + remoteUsername + " -perm -664 -type f";
} else {
findCmd = "find " + remoteDirPath + " -maxdepth 1 -type f";
}
findCmd = findCmd + " -printf '%T@\t%s\t%p\n' 2> /dev/null";
ShellCommandResult sshResult = sshCommand.execShellCommand(findCmd);
String[] allFiles = sshResult.getResult().split("\n");
LogFileFilter baseFileFilter = new LogFileFilter(baseName);
for (String remoteFileItem : allFiles) {
String[] remoteFileItems = remoteFileItem.split("\t");
if (remoteFileItems.length != 3) {
logger.warn("Got remote file entry, but not formatted as expected: {}", remoteFileItem);
} else {
String remoteFilePath = remoteFileItems[2];
File remoteLogFile = new File(remoteFilePath);
//logger.debug("Consider {} vs {}", remoteLogFile.getName(), filenameRegex);
if (baseFileFilter.accept(remoteLogFile)) {
matches.add(new FileInfo(remoteFilePath, remoteFileItems[1], remoteFileItems[0]));
//logger.debug("{} matches!", remoteLogFile.getName());
} else {
logger.trace ("No match between regex '{}' and '{}'", baseName, remoteLogFile.getName());
}
}
}
return matches;
}
/**
* Formats the filename for a copied log file to enable identification and sorting.
* Currently transforms rolling-appender style (".1") file suffixes to
* daily-rolling-appender style ("yyyy-MM-dd-HH-ss)".
*
* @param logFileInfo FileInfo structure for log file
* @param localTargetDir The directory where this file will be placed
* @return New filename (no path) for log file
*/
private static String targetNameForLogFile(FileInfo logFileInfo, File localTargetDir)
{
// Assume log files have either a rolling file extension (e.g. ".1")
// or a daily-rolling file extension ("2012-05-01-14-00")
// Convert log files with rolling extensions to dailyRolling for uniqueness
File logFile = new File(logFileInfo.filePath);
String localName = logFile.getName();
if (localName.matches(LoggingUtil.rollingFilePatternString)) {
long modTime = (long) (Double.parseDouble(logFileInfo.fileModtime) * 1000.0);
Date modDate = new Date(modTime);
DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern);
String formattedDate = format.format(modDate);
localName = localName.replaceFirst(LoggingUtil.rollingFilePatternString, "$1." + formattedDate);
// Uniquify in target directory
// Don't uniquify if we don't delete source logs, since this will result in local duplicates
if (uniquifyCopiedFilenames) {
int id = 0;
File targetFile = new File(localTargetDir, localName);
boolean uniquified = false;
while (targetFile.exists()) {
if (++id > uniquifyLimit) {
break;
}
uniquified = true;
targetFile = new File(localTargetDir, localName + "." + String.valueOf(id));
}
if (uniquified) {
if(logger.isDebugEnabled()) {
logger.debug("Uniquified filename {} to {}", logFile.getName(), targetFile.getName());
}
}
localName = targetFile.getName();
}
if(logger.isDebugEnabled()) {
logger.debug("Changed target filename {} to {}", logFile.getName(), localName);
}
}
return localName;
}
/**
* Formats the filename for a copied log file to enable identification and sorting.
* Currently transforms rolling-appender style (".1") file suffixes to
* daily-rolling-appender style ("yyyy-MM-dd-HH-ss)".
*
* @param logFile Log-file File object
* @param localTargetDir The directory where this file will be placed
* @return New filename (no path) for log file
*/
private static String targetNameForLogFile(File logFile, File localTargetDir)
{
// Assume log files have either a rolling file extension (e.g. ".1")
// or a daily-rolling file extension ("2012-05-01-14-00")
// Convert log files with rolling extensions to dailyRolling for uniqueness
String localName = logFile.getName();
if (localName.matches(LoggingUtil.rollingFilePatternString)) {
long modTime = logFile.lastModified();
Date modDate = new Date(modTime);
DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern);
String formattedDate = format.format(modDate);
localName = localName.replaceFirst(LoggingUtil.rollingFilePatternString, "$1." + formattedDate);
// Uniquify in target directory, requested
// Don't uniquify if we don't delete the original (source) log file,
// since we'll end up with duplicates
if (uniquifyCopiedFilenames) {
int id = 0;
File targetFile = new File(localTargetDir, localName);
while (targetFile.exists()) {
if (++id > uniquifyLimit) {
break;
}
targetFile = new File(localTargetDir, localName + "." + String.valueOf(id));
if(logger.isDebugEnabled()) {
logger.debug("Uniquified filename {} to {}", logFile.getName(), targetFile.getName());
}
}
localName = targetFile.getName();
}
if(logger.isDebugEnabled()) {
logger.debug("Changed target filename {} to {}", logFile.getName(), localName);
}
}
return localName;
}
/**
* Copies a file on a remote host to a local path. Before copying, renames source file
* with a temporary name to move it out of the way of any rotator process.
* Optionally deletes the file from the remote host.
*
* @param sshCommand An active (connected) SSHCommand
* @param remoteFileInfo A FileInfo structure representing the remote file
* @param localFilePath The full path for the local file copy
* @param deleteSource If <code>true</code> and the copied succeeds, the remote file will be deleted
* @return The number of bytes copied.
* @throws JSchException
* @throws IOException
*/
private static long copyRemoteFile(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath, boolean deleteSource) throws JSchException, IOException
{
String remoteFilePath = remoteFileInfo.filePath;
String tempRemotePath = tempNameForSourceFile(remoteFilePath);
File localFile = null;
long bytesCopied = 0L;
IOException cleanupIOException = null;
JSchException cleanupJSchException = null;
boolean success = false;
// Copy file and validate result
try {
// Rename the remote file with a temporary name before copying
if(logger.isDebugEnabled()) {
logger.debug("Renaming remote file to: {}", tempRemotePath);
}
sshCommand.execShellCommand("mv " + remoteFilePath + " " + tempRemotePath);
// Copy the (renamed) remote file to local destination
remoteFileInfo.filePath = tempRemotePath; // remote file name is needed during validation
bytesCopied = sshCommand.copyFrom(tempRemotePath, localFilePath);
localFile = new File(localFilePath);
long modTime = (long) (Double.parseDouble(remoteFileInfo.fileModtime) * 1000.0);
localFile.setLastModified(modTime);
// Check integrity of copy
success = validateRemoteFileCopy(sshCommand, remoteFileInfo, localFilePath);
if (!success) {
logger.warn("Copy of file {} did not pass integrity check!", remoteFilePath);
}
} catch (IOException ex) {
// If there's been an error copying the file, we may be left with a zero-length or incomplete file
if ((localFile != null) && localFile.exists() && !success) {
if(logger.isDebugEnabled()) {
logger.debug("Deleting failed local copy of remote file: {}", localFile.getAbsolutePath());
}
localFile.delete();
}
} finally {
// Use a try-block during cleanup, but only throw exception if the
// main file-copy try-block doesn't
try {
if (deleteSource && success) {
if(logger.isDebugEnabled()) {
logger.debug("Deleting remote file: {}", tempRemotePath);
}
sshCommand.execShellCommand("rm " + tempRemotePath);
} else {
// Move source file back from temporary name
sshCommand.execShellCommand("mv " + tempRemotePath + " " + remoteFilePath);
}
} catch (JSchException ex) {
logger.warn("JSchException during remote file copy cleanup: {}", ex);
cleanupJSchException = new JSchException(ex.getMessage());
} catch (IOException ex) {
logger.warn("IOException during remote file copy cleanup: {}", ex);
cleanupIOException = new IOException(ex);
}
remoteFileInfo.filePath = remoteFilePath; // restore original file name in argument
}
if (cleanupJSchException != null) {
throw cleanupJSchException;
} else if (cleanupIOException != null) {
throw cleanupIOException;
}
return bytesCopied;
}
private static boolean validateRemoteFileCopy(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath) throws JSchException, IOException
{
boolean success;
File localFile = new File(localFilePath);
String remoteFilePath = remoteFileInfo.filePath;
long remoteLength = Long.parseLong(remoteFileInfo.fileSize);
long localLength = localFile.length();
if (localLength != remoteLength) {
logger.warn("File-size difference after copy of remote log file {}: {}",
remoteFilePath, "remote size: " + String.valueOf(remoteLength) + ", local size: " + String.valueOf(localLength));
success = false;
} else {
// Check if last lines of files are the same
String remoteTail = execTailOnRemoteFile(sshCommand, remoteFilePath, 1);
String localTail = execTailOnFile(localFile, 1);
success = remoteTail != null && remoteTail.equals(localTail);
if (!success) {
logger.error("Last lines of copied files are different: '{}' vs '{}'", localTail, remoteTail);
}
}
return success;
}
/**
* Copies a file on a remote host to a local path, using a temporary local file intermediary. Optionally deletes the file from the remote host.
*
* @param sshCommand An active (connected) SSHCommand
* @param remoteFileInfo A FileInfo structure representing the remote file
* @param localFilePath The full path for the local file copy
* @param deleteSource If <code>true</code> and the copied succeeds, the remote file will be deleted
* @return The number of bytes copied.
* @throws JSchException
* @throws IOException
*/
private static long copyRemoteFileWithWorkingTemp(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath, boolean deleteSource) throws JSchException, IOException
{
String tempLocalPath = tempNameForWorkingFile(localFilePath);
long bytesCopied = copyRemoteFile(sshCommand, remoteFileInfo, tempLocalPath, deleteSource);
(new File(tempLocalPath)).renameTo(new File(localFilePath));
return bytesCopied;
}
private static String execTailOnRemoteFile(SSHCommand sshCommand, String remoteFilePath, int numLines) throws JSchException, IOException
{
if ((sshCommand == null) || !sshCommand.isConnected()) {
throw new JSchException("Not connected with a valid SSH session");
}
String numLinesArg = String.valueOf(numLines);
if ((numLines < 0) || (numLinesArg == null) || (numLinesArg.length() == 0) || (remoteFilePath == null) || (remoteFilePath.length() == 0)) {
return null;
} else if (numLines == 0) {
return "";
}
String tailCmd = "/usr/bin/tail -n " + numLinesArg + " " + remoteFilePath;
ShellCommandResult sshResult = sshCommand.execShellCommand(tailCmd);
if(logger.isDebugEnabled()) {
logger.debug("Remote tail process exited with code {}", String.valueOf(sshResult.getExitStatus()));
}
String logLines = sshResult.getResult();
return logLines;
}
/**
* Format an ip address string for use as a directory name.
*
* @param ipAddress The ip address in the usual IPv4 format
* @return
*/
private static String ipToCollectionDirectory(String ipAddress)
{
return ipAddress.replace(".", "_");
}
/**
* Reverses the formatting performed by the <code>ipToCollectionDirectory</code> function.
*
* @param dirName The collection directory name
* @return The name re-formatted as an IPv4 address
*/
private static String collectionDirectoryToIP(String dirName)
{
String ip = dirName.replaceAll("^(\\d{1,3})_(\\d{1,3})_(\\d{1,3})_(\\d{1,3})$", "$1\\.$2\\.$3\\.$4");
return ip;
}
private static String tempNameForWorkingFile(String fileNameOrPath)
{
return fileNameOrPath + workingFileSuffix;
}
private static String tempNameForSourceFile(String fileNameOrPath)
{
Date modDate = new Date();
DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern);
String formattedDate = format.format(modDate);
return fileNameOrPath + ".collected_" + formattedDate;
}
/**
* Log an entire Instance structure.
*
* @param instance The Instance to send to the log
*/
public static void logInstance(Instance instance)
{
if(logger.isDebugEnabled()) {
logger.debug("Got instance: {}", instance);
}
logger.trace("SSHKeyName: {}", instance.getSSHKeyName());
logger.trace("User: {}", instance.getUser());
logger.trace("Password: {}", instance.getPassword());
logger.trace("Port: {}", instance.getPort());
logger.trace("Area: {}", instance.getArea());
logger.trace("SSHKey: {}", instance.getSSHKey());
}
/**
* Gets a specified number of most-recent log lines from the localhost's
* rotated logs.
*
* This method has been supplanted by the other log-collection methods in this class.
*
* @param numLines
* @return
* @throws IOException
*/
public static String getLocalLogLines(LogFileSource logSource, int numLines) throws IOException
{
if (!LogFileSource.JAVA.equals(logSource) && !LogFileSource.SMX.equals(logSource)) {
logger.warn("Unexpected log-source value: {}", logSource == null ? "(null)" : logSource);
return null;
}
StringBuilder logLines = null;
String logFilePath = LoggingUtil.getLocalLogFilePath(logSource);
int lineCount = 0;
File logDir = (new File(logFilePath)).getParentFile();
if (!logDir.exists() || !logDir.isDirectory()) {
return null;
}
// Try getting lines from the current log file(s) ...
logLines = new StringBuilder();
int logGen = 1;
try {
while (lineCount < numLines) {
File logFile = new File(logFilePath + "." + String.valueOf(logGen));
if (!logFile.exists()) {
break;
}
String logContent = getTailOfFile(logFile, numLines - lineCount);
logLines.insert(0, logContent);
lineCount += lineCount(logContent);
logGen++;
}
} catch (IOException ex) {
logger.warn("Couldn't read from log file {}", logFilePath);
}
return LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, E3Constant.localhost, logLines.toString());
}
}