* @param inputControlFile stream of control.json file content
* @return JobStatus containing success or error information
*/
private static JobStatus publishViaFTPDropboxV2(final UserPreferences userPrefs, final String datasetId,
final File csvOrTsvFile, final InputStream inputControlFile) {
JobStatus status = JobStatus.PUBLISH_ERROR;
String ftpHost;
try {
ftpHost = getFTPHost(userPrefs);
} catch (Exception e) {
e.printStackTrace();
status.setMessage("Error obtaining FTP host: " + e.getMessage());
return status;
}
FTPSClient ftp = null;
try {
ftp = new FTPSClient(false, SSLContext.getDefault());
System.out.println("Connecting to " + ftpHost + ":" + FTP_HOST_PORT);
// ADDED connection retry logic
int tryCount = 0;
int maxTries = 5;
boolean connectionSuccessful = false;
do {
try {
ftp.connect(ftpHost, FTP_HOST_PORT);
connectionSuccessful = true;
} catch (Exception connectException) {
// wait 2 secs, then retry connection
try {
Thread.sleep((long) (Math.pow(2, (tryCount + 1)) * 1000));
} catch (InterruptedException e) {
// do nothing
}
}
if (++tryCount > maxTries) {
status.setMessage("FTP server refused connection (connection timeout).");
return status;
}
} while(!connectionSuccessful);
// END connection retry logic
SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo();
ftp.login(connectionInfo.getUser(), connectionInfo.getPassword());
// verify connection was successful
if(FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
System.out.println("ftp.setFileType(FTP.BINARY_FILE_TYPE)");
ftp.setFileType(FTP.BINARY_FILE_TYPE);
System.out.println("ftp.enterLocalPassiveMode()");
ftp.enterLocalPassiveMode();
// Set protection buffer size (what does this do??)
//ftp.execPBSZ(0);
// Set data channel protection to private
System.out.println("ftp.execPROT(\"P\")");
ftp.execPROT("P");
String pathToDomainRoot = getPathToDomainRoot(ftp, connectionInfo);
String pathToDatasetDir = pathToDomainRoot + "/" + datasetId;
// if datasetId does not exist then create the directory
System.out.println("ftp.listFiles(" + pathToDatasetDir + "/" + FTP_STATUS_FILENAME + ")");
FTPFile[] checkDatasetDirExists = ftp.listFiles(pathToDatasetDir + "/" + FTP_STATUS_FILENAME);
if(checkDatasetDirExists.length == 0) {
System.out.println("ftp.makeDirectory(" + pathToDatasetDir + ")");
boolean datasetDirCreated = ftp.makeDirectory(pathToDatasetDir);
if(!datasetDirCreated) {
closeFTPConnection(ftp);
status.setMessage("Error creating dataset ID directory at" +
" '" + pathToDatasetDir + "': " + ftp.getReplyString());
return status;
}
}
// set request Id for control file upload
String controlFileRequestId = setFTPRequestId(ftp, pathToDomainRoot + "/" + FTP_REQUEST_ID_FILENAME);
if(controlFileRequestId.startsWith(FAILURE_PREFIX)) {
closeFTPConnection(ftp);
status.setMessage("Error setting request Id: " + controlFileRequestId);
return status;
}
// for tracking DataSync version
recordDataSyncVersion(ftp, pathToDomainRoot + "/" + FTP_DATASYNC_VERSION_FILENAME);
// upload control.json file content
String controlFilePathFTP = pathToDatasetDir + "/" + FTP_CONTROL_FILENAME;
String controlResponse = uploadAndEnqueue(ftp, inputControlFile, controlFilePathFTP, 0);
inputControlFile.close();
if(!controlResponse.equals(SUCCESS_PREFIX)) {
closeFTPConnection(ftp);
status.setMessage("Error uploading control file: " + controlResponse);
return status;
}
// ensure control.json was uploaded without issues
String controlFileUploadStatus = pollUploadStatus(
ftp, pathToDatasetDir + "/" + FTP_STATUS_FILENAME, controlFileRequestId);
if(!controlFileUploadStatus.startsWith(SUCCESS_PREFIX)) {
closeFTPConnection(ftp);
status.setMessage("Error uploading control file: " + controlFileUploadStatus);
return status;
}
System.out.println("Publishing entire file via FTPS...");
// set request Id for this job
String csvOrTsvFileRequestId = setFTPRequestId(ftp, pathToDomainRoot + "/" + FTP_REQUEST_ID_FILENAME);
if(csvOrTsvFileRequestId.startsWith(FAILURE_PREFIX)) {
closeFTPConnection(ftp);
status.setMessage("Error setting request Id: " + csvOrTsvFileRequestId);
return status;
}
// attempt to gzip CSV/TSV file before uploading
boolean deleteFileToUpload = false;
File fileToUpload;
String dataFilePathFTP;
try {
System.out.println("Gzipping file before uploading...");
fileToUpload = createTempGzippedFile(csvOrTsvFile);
dataFilePathFTP = pathToDatasetDir + "/" + csvOrTsvFile.getName() + ".gz";
deleteFileToUpload = true;
} catch (IOException ex) {
// if gzipping fails revert to sending raw CSV
System.out.println("Gzipping failed, uploading CSV directly");
fileToUpload = csvOrTsvFile;
dataFilePathFTP = pathToDatasetDir + "/" + csvOrTsvFile.getName();
}
// upload CSV/TSV file
long dataFileSizeBytes = fileToUpload.length();
InputStream inputDataFile = new FileInputStream(fileToUpload);
String dataFileResponse = uploadAndEnqueue(ftp, inputDataFile, dataFilePathFTP, dataFileSizeBytes);
inputDataFile.close();
if(deleteFileToUpload)
fileToUpload.delete();
if(!dataFileResponse.equals(SUCCESS_PREFIX)) {
closeFTPConnection(ftp);
status.setMessage(dataFileResponse);
return status;
}
// Poll upload status until complete
String dataFileUploadStatus = pollUploadStatus(
ftp, pathToDatasetDir + "/" + FTP_STATUS_FILENAME, csvOrTsvFileRequestId);
if(!dataFileUploadStatus.startsWith(SUCCESS_PREFIX)) {
status.setMessage(dataFileUploadStatus);
return status;
}
} else {
status.setMessage("FTP server refused connection (check your username and password).");
return status;
}
} catch(IOException e) {
e.printStackTrace();
status.setMessage("FTP error: " + e.getMessage());
return status;
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
status.setMessage("Java error: " + e.getMessage());
return status;
} finally {
if(ftp != null)
closeFTPConnection(ftp);
}