public void onReceive(Object message) {
if(message instanceof RemoteFile) {
RemoteFile zipfile = (RemoteFile) message;
try {
URL url = new URL(zipfile.url.toString().replace("https", "http"));
InputStream stream = url.openStream();
byte[] buf = new byte[1024];
ZipInputStream zipinputstream = null;
ZipEntry entry;
zipinputstream = new ZipInputStream(stream);
Integer id = 0;
while((entry = zipinputstream.getNextEntry()) != null) {
id++;
Crawler.logger.info("Extracting: " + entry);
String format = entry.getName().substring(entry.getName().lastIndexOf('.') + 1);
if(!FileReader.handles(format)) {
continue;
}
File tmpFile = new File("tmp/" + entry.getName().replace("/", "-"));
FileOutputStream fos = new FileOutputStream(tmpFile);
int data;
while (0 < (data = zipinputstream.read(buf))){
fos.write(buf, 0, data);
}
fos.close();
zipinputstream.closeEntry();
Crawler.logger.info("Completed extraction for: " + entry);
RemoteFile file = new RemoteFile("file://" + tmpFile.getAbsolutePath());
ICSVFetcherResult callback = new ElasticIndexer(client, file.type, file.id + "-" + id);
IFormatReader fileReader = FileReader.read(file, callback);
if(fileReader != null) {