/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.chukwa.extraction.engine.datasource.record;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
import org.apache.hadoop.chukwa.extraction.engine.Record;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class ChukwaFileParser
{
static final int timestampField = 0;
@SuppressWarnings("deprecation")
public static List<Record> readData(String cluster,String dataSource,int maxRows,long t1, long t0,
long maxOffset,String filter,String fileName,FileSystem fs ) throws
MalformedFileFormat
{
//String source = "NameNode." + fileName;
List<Record> records = new LinkedList<Record>();
FSDataInputStream dataIS = null;
int lineCount = 0;
try
{
if (!fs.exists(new Path(fileName)))
{
System.out.println("fileName not there!");
return records;
}
System.out.println("NameNodeParser Open [" +fileName + "]");
dataIS = fs.open(new Path(fileName));
System.out.println("NameNodeParser Open2 [" +fileName + "]");
long timestamp = 0;
int listSize = 0;
String line = null;
String[] data = null;
long offset = 0;
do
{
offset = dataIS.getPos();
// Need TODO something here
// if (offset > maxOffset)
// {
// break;
// }
line = dataIS.readLine();
lineCount ++;
// System.out.println("NameNodeParser Line [" +line + "]");
if (line != null)
{
//empty lines
if (line.length() < 14)
{
// System.out.println("NameNodeParser Line < 14! [" +line + "]");
continue;
}
// System.out.println("Line [" +line + "]");
data = line.split("\t");// Default separator for TextOutputFormat!
try
{
timestamp = Long.parseLong(data[timestampField]);
} catch (Exception e)
{
e.printStackTrace();
//throw new MalformedFileFormat(e);
}
if (timestamp < t0)
{
// System.out.println("Line not in range. Skipping: " +line);
// System.out.println("Search for: " + new Date(t0) + " is :" + new Date(timestamp));
continue;
}
else if ((timestamp < t1) && (offset < maxOffset )) //JB (epochTS < maxDate)
{
// System.out.println("In Range: " + line);
boolean valid = false;
if ( (filter == null || filter.equals("") ))
{
valid = true;
}
else if (line.indexOf(filter) > 0)
{
valid = true;
}
if (valid)
{
// System.out.println("In Range In Filter: " + line);
ChukwaRecord record = new ChukwaRecord();
record.setTime(timestamp);
record.add("offset", ""+offset);
record.add(Record.bodyField, data[1]);
record.add(Record.sourceField, dataSource);
records.add(record);
listSize = records.size();
if (listSize > maxRows)
{
records.remove(0);
// System.out.println("==========>>>>>REMOVING: " + e);
}
}
else
{
// System.out.println("In Range ==================>>>>>>>>> OUT Regex: " + line);
}
}
else
{
// System.out.println("Line out of range. Stopping now: " +line);
break;
}
}
} while (line != null);
}
catch(Exception e)
{
e.printStackTrace();
}
finally
{
System.out.println("File: " +fileName +" Line count: " + lineCount);
try
{dataIS.close();}
catch (IOException e)
{}
}
return records;
}
}