writeFirstRecord(os, "test", timestamp);
List<TestRecord> records = buildTestRecords(BASIC_TEST_RECORD_COUNT);
long testAttemptTime = System.currentTimeMillis();
for (TestRecord record : records) {
NIOHttpHeaders headers = new NIOHttpHeaders();
for (int i=0;i<record.headers.size();++i) {
headers.set(record.headers.get(i).e0,record.headers.get(i).e1);
}
write(os,record.url,"test",1,1,record.data,0,record.data.length,headers,"text/html",MD5Hash.digest(record.data).toString(),12345,testAttemptTime);
}
os.flush();
os.close();
final AtomicBoolean streamClosed = new AtomicBoolean();
// setup ArcFileReader to read the file
InputStream in = new ByteArrayInputStream(os.getData(),0,os.getLength()) {
public synchronized int read(byte b[], int off, int len) {
len = 1;
return super.read(b, off, len);
}
public void close() throws IOException {
super.close();
streamClosed.set(true);
}
};
ARCFileReader reader = new ARCFileReader(in);
int index = 0;
Text key = new Text();
BytesWritable value = new BytesWritable();
// iterate and validate stuff ...
while (reader.hasMoreItems()) {
reader.nextKeyValue(key, value);
TestRecord testRecord = records.get(index++);
// get test key bytes as utf-8 bytes ...
byte[] testKeyBytes = testRecord.url.getBytes(Charset.forName("UTF-8"));
// compare against raw key bytes to validate key is the same (Text's utf-8 mapping code replaces invalid characters
// with ?, which causes our test case (which does use invalid characters to from the key, to break.
Assert.assertTrue(compareTo(testKeyBytes,0,testKeyBytes.length,key.getBytes(),0,key.getLength()) == 0);
// retured bytes represent the header(encoded in utf-8), terminated by a \r\n\r\n. The content follows this terminator
// we search for this specific byte pattern to locate start of content, then compare it against source ...
int indexofHeaderTerminator = ByteArrayUtils.indexOf(value.getBytes(), 0, value.getLength(), "\r\n\r\n".getBytes());
if (indexofHeaderTerminator == -1) {
throw new IOException("No Header Terminator found in Value!");
}
indexofHeaderTerminator += 4;
// read headers ...
String headersText = new String(value.getBytes(),0,indexofHeaderTerminator,Charset.forName("UTF-8"));
NIOHttpHeaders headers = NIOHttpHeaders.parseHttpHeaders(headersText);
for (int i=0;i<testRecord.headers.size();++i) {
Pair<String,String> testHeaderRecord = testRecord.headers.get(i);
Assert.assertNotNull(headers.findValue(testHeaderRecord.e0));
Assert.assertEquals(testHeaderRecord.e1,headers.findValue(testHeaderRecord.e0));
}
Assert.assertTrue(compareTo(testRecord.data,0,testRecord.data.length,value.getBytes(),indexofHeaderTerminator,testRecord.data.length) == 0);
}
reader.close();