/*
* Copyright 2014 Steven Jennison https://kd8rho.net, and contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.kd8rho.util;
import net.kd8rho.util.data.HTTPError;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.*;
import java.util.ArrayList;
/**
* Web class designed to allow for easy downloading of data from Web sites.
* Created by Steven on 1/30/14. Package: net.kd8rho.util Project: Util
* <p/>
*
* @author Steven Jennison
* @version 1.0
*/
public class Web {
public static final String TIMED_OUT = "Read Request Timed Out";
public static SJIO io = new SJIO();
public static HTTPError last_error;
/**
* Gets text from URL and returns it as an ArrayList<String>
*
* @param Url the URL to get data from
* @param retries number of times to retry when ioexception recieved while getting data
* @param showProgress Boolean to show progress or not
* @param progressChar Progress character
* @param numCharsForNewline number of characters to draw before dropping to a newline
* @return an ArrayList of Strings containing the text of the webpage, or null if it failed more than "retries"
* @throws java.net.MalformedURLException If the URL given is not a valid URL
*/
public static ArrayList<String> getURLTextToArray(String Url, int retries, boolean showProgress, String progressChar, int numCharsForNewline) throws MalformedURLException {
last_error=null;
int count = 0;
boolean success = false;
ArrayList<String> httpResponseData = new ArrayList<>();
while (count < 3 && !success) {
URL resource;
httpResponseData = new ArrayList<>();
try {
resource = new URL(Url);
} catch (MalformedURLException e) {
io.log(Web.class.getName(), "getURLTextToArray", "Invalid URL", SJIO.LEVEL.ERROR);
throw new MalformedURLException(e.getMessage());
}
BufferedReader in;
try {
io.log(Web.class.getName(), "getURLTextToArray", "Getting data from URL " + Url, SJIO.LEVEL.FINE);
in = new BufferedReader(
new InputStreamReader(resource.openStream()));
String inputLine;
int numChars = 0;
while ((inputLine = in.readLine()) != null) {
if (showProgress) {
if (numChars <= numCharsForNewline) {
numChars = 0;
io.write("\n");
}
io.write(progressChar);
}
httpResponseData.add(inputLine);
}
in.close();
success = true;
io.write("\n");
} catch (IOException e) {
if(e.getMessage().contains("HTTP response code"))
{
String[] splitString = e.getMessage().split(" ");
String errorCode="0";
for (int i = 0; i < splitString.length; i++) {
String s = splitString[i];
if (s.equals("code:")) {
errorCode = splitString[i + 1];
break;
}
}
HTTPError theError=new HTTPError(Integer.parseInt(errorCode),e.getMessage());
last_error=theError;
if(theError.getErrorCode()==403||theError.getErrorCode()==404)
{
break;
}
}
io.writeLine("Failed with " + e.getClass().getName() + ". Retrying " + (retries - count) + " more times");
io.log(Web.class.getName(), "getURLTextToArray", e.getClass().getName() + "\n" + e.getMessage(), SJIO.LEVEL.WARN);
count++;
}
}
if (!success) {
io.log(Web.class.getName(), "getURLTextToArray", "Failed permanently trying get data", SJIO.LEVEL.WARN);
if(last_error!=null)
{
io.log(Web.class.getName(), "getURLTextToArray", "Error code: "+last_error.getErrorCode(), SJIO.LEVEL.WARN);
}
return null;
}
return httpResponseData;
}
/**
* Gets text from URL and returns it as an ArrayList<String>
*
* @param Url the URL to get data from
* @return an ArrayList of Strings containing the text of the webpage, or null if it failed more than 3 times
* @throws java.net.MalformedURLException If the URL given is not a valid URL
*/
public static ArrayList<String> getURLTextToArray(String Url) throws MalformedURLException {
return getURLTextToArray(Url, 3, false, " ", 1);
}
/**
* Gets text from a URL protected by HTTP authentication, using the given username and password
*
* @param Url the URL to get data from
* @param username the username to logon with
* @param password the password to logon with
* @return an ArrayList of Strings containing the text of the webpage, or null if it failed more than 3 times
* @throws java.net.MalformedURLException If the URL given is not a valid URL
*/
public static ArrayList<String> getURLTextToArray(String Url, final String username, final String password) throws MalformedURLException {
return getURLTextToArray(Url, username, password, 3, false, " ", 1);
}
/**
* Gets text from a URL protected by HTTP authentication, using the given username and password
*
* @param Url the URL to get data from
* @param username the username to logon with
* @param password the password to logon with
* @param retries number of times to retry when ioexception recieved while getting data
* @param showProgress Boolean to show progress or not
* @param progressChar Progress character
* @param numCharsForNewline number of characters to draw before dropping to a newline
* @return an ArrayList of Strings containing the text of the webpage, or null if it failed more than 3 times
* @throws java.net.MalformedURLException If the URL given is not a valid URL
*/
public static ArrayList<String> getURLTextToArray(String Url, final String username, final String password, int retries, boolean showProgress, String progressChar, int numCharsForNewline) throws MalformedURLException {
last_error=null;
int count = 0;
boolean success = false;
Authenticator.setDefault(new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password.toCharArray());
}
});
ArrayList<String> httpResponseData = new ArrayList<>();
while (count < 3 && !success) {
HttpURLConnection c = null;
httpResponseData = new ArrayList<>();
try {
c = (HttpURLConnection) new URL(Url).openConnection();
c.setUseCaches(false);
c.connect();
} catch (MalformedURLException e) {
io.log(Web.class.getName(), "getURLTextToArray", "Invalid URL", SJIO.LEVEL.ERROR);
throw new MalformedURLException(e.getMessage());
} catch (IOException e) {
e.printStackTrace();
}
BufferedReader in;
try {
io.log(Web.class.getName(), "getURLTextToArray", "Getting data from URL " + Url, SJIO.LEVEL.FINE);
assert c != null;
in = new BufferedReader(
new InputStreamReader(c.getInputStream()));
String inputLine;
int numChars = 0;
while ((inputLine = in.readLine()) != null) {
if (showProgress) {
if (numChars <= numCharsForNewline) {
numChars = 0;
io.write("\n");
}
io.write(progressChar);
}
httpResponseData.add(inputLine);
}
in.close();
success = true;
io.write("\n");
} catch (Exception e) {
if(e.getMessage().contains("HTTP response code"))
{
String[] splitString = e.getMessage().split(" ");
String errorCode="0";
for (int i = 0; i < splitString.length; i++) {
String s = splitString[i];
if (s.equals("code:")) {
errorCode = splitString[i + 1];
break;
}
}
HTTPError theError=new HTTPError(Integer.parseInt(errorCode),e.getMessage());
last_error=theError;
if(theError.getErrorCode()==403||theError.getErrorCode()==404)
{
break;
}
}
io.writeLine("Failed with " + e.getClass().getName() + ". Retrying " + (retries - count) + " more times");
io.log(Web.class.getName(), "getURLTextToArray", e.getClass().getName() + "\n" + e.getMessage(), SJIO.LEVEL.WARN);
count++;
}
}
if (!success) {
io.log(Web.class.getName(), "getURLTextToArray", "Failed permanently trying get data", SJIO.LEVEL.WARN);
if(last_error!=null)
{
io.log(Web.class.getName(), "getURLTextToArray", "Error code: "+last_error.getErrorCode(), SJIO.LEVEL.WARN);
}
return null;
}
return httpResponseData;
}
}