/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.sqoop.manager;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.PreparedStatement;
import java.util.Arrays;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
import com.cloudera.sqoop.TestExport;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
/**
* Test the PGBulkloadManager implementations.
* PGBulkloadManager uses both JDBC driver and pg_bulkload to facilitate it.
*
* Since this requires a Postgresql installation on your local machine to use,
* this class is named in such a way that Hadoop's default QA process does not
* run it.
*
* You need to run this manually with -Dtestcase=PGBulkloadManagerManualTest.
*
* You need to put Postgresql's JDBC driver library into lib dir.
*
* You need to create a sqooptest superuser and database and tablespace,
* and install pg_bulkload for sqooptest database:
*
* $ sudo -u postgres createuser -U postgres -s sqooptest
* $ sudo -u postgres createdb -U sqooptest sqooptest
* $ sudo -u postgres mkdir /var/pgdata/stagingtablespace
* $ psql -U sqooptest
* -f /usr/local/share/postgresql/contrib/pg_bulkload.sql sqooptest
* $ psql -U sqooptest sqooptest
* sqooptest=# CREATE USER sqooptest;
* sqooptest=# CREATE DATABASE sqooptest;
* sqooptest=# CREATE TABLESPACE sqooptest
* LOCATION '/var/pgdata/stagingtablespace';
* sqooptest=# \q
*
*/
public class PGBulkloadManagerManualTest extends TestExport {
public static final Log LOG =
LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
private DBConfiguration dbConf;
static final String HOST_URL =
System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE =
System.getProperty("sqoop.test.postgresql.database", "sqooptest");
static final String TABLESPACE =
System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
static final String USERNAME =
System.getProperty("sqoop.test.postgresql.username", "sqooptest");
static final String PG_BULKLOAD =
System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
static final String CONNECT_STRING = HOST_URL + DATABASE;
public PGBulkloadManagerManualTest() {
JobConf conf = new JobConf(getConf());
DBConfiguration.configureDB(conf,
"org.postgresql.Driver",
getConnectString(),
getUserName(),
(String) null, (Integer) null);
dbConf = new DBConfiguration(conf);
}
@Override
protected boolean useHsqldbTestServer() {
return false;
}
@Override
protected String getConnectString() {
return CONNECT_STRING;
}
protected String getUserName() {
return USERNAME;
}
@Override
protected String getTablePrefix() {
return super.getTablePrefix().toLowerCase();
}
@Override
protected String getTableName() {
return super.getTableName().toLowerCase();
}
@Override
public String getStagingTableName() {
return super.getStagingTableName().toLowerCase();
}
@Override
protected Connection getConnection() {
try {
Connection conn = dbConf.getConnection();
conn.setAutoCommit(false);
PreparedStatement stmt =
conn.prepareStatement("SET extra_float_digits TO 0");
stmt.executeUpdate();
conn.commit();
return conn;
} catch (SQLException sqlE) {
LOG.error("Could not get connection to test server: " + sqlE);
return null;
} catch (ClassNotFoundException cnfE) {
LOG.error("Could not find driver class: " + cnfE);
return null;
}
}
@Override
protected String getDropTableStatement(String tableName) {
return "DROP TABLE IF EXISTS " + tableName;
}
@Override
protected String[] getArgv(boolean includeHadoopFlags,
int rowsPerStatement,
int statementsPerTx,
String... additionalArgv) {
ArrayList<String> args =
new ArrayList<String>(Arrays.asList(additionalArgv));
args.add("-D");
args.add("pgbulkload.bin=" + PG_BULKLOAD);
args.add("--username");
args.add(getUserName());
args.add("--connection-manager");
args.add("org.apache.sqoop.manager.PGBulkloadManager");
args.add("--staging-table");
args.add("dummy");
args.add("--clear-staging-table");
return super.getArgv(includeHadoopFlags,
rowsPerStatement,
statementsPerTx,
args.toArray(new String[0]));
}
@Override
protected String [] getCodeGenArgv(String... extraArgs) {
ArrayList<String> args = new ArrayList<String>(Arrays.asList(extraArgs));
args.add("--username");
args.add(getUserName());
return super.getCodeGenArgv(args.toArray(new String[0]));
}
@Override
public void testColumnsExport() throws IOException, SQLException {
// PGBulkloadManager does not support --columns option.
}
public void testMultiReduceExport() throws IOException, SQLException {
String[] genericargs = newStrArray(null, "-Dmapred.reduce.tasks=2");
multiFileTestWithGenericArgs(2, 10, 2, genericargs);
}
public void testMultiReduceExportWithNewProp() throws IOException, SQLException {
String[] genericargs = newStrArray(null, "-Dmapreduce.job.reduces=2");
multiFileTestWithGenericArgs(2, 10, 2, genericargs);
}
public void testExportWithTablespace() throws IOException, SQLException {
String[] genericargs =
newStrArray(null, "-Dpgbulkload.staging.tablespace=" + TABLESPACE);
multiFileTestWithGenericArgs(1, 10, 1, genericargs);
}
protected void multiFileTestWithGenericArgs(int numFiles,
int recordsPerMap,
int numMaps,
String[] genericargs,
String... argv)
throws IOException, SQLException {
final int TOTAL_RECORDS = numFiles * recordsPerMap;
try {
LOG.info("Beginning test: numFiles=" + numFiles + "; recordsPerMap="
+ recordsPerMap + "; numMaps=" + numMaps);
LOG.info(" with genericargs: ");
for (String arg : genericargs) {
LOG.info(" " + arg);
}
for (int i = 0; i < numFiles; i++) {
createTextFile(i, recordsPerMap, false);
}
createTable();
runExport(getArgv(true, 10, 10,
newStrArray(newStrArray(genericargs, argv),
"-m", "" + numMaps)));
verifyExport(TOTAL_RECORDS);
} finally {
LOG.info("multi-reduce test complete");
}
}
}