Package com.cloudera.sqoop

Examples of com.cloudera.sqoop.SqoopOptions

197
198
199
200
201
202
203
204
205
206
207
208
209
210
  }

  private void handleNetezzaImportExtraArgs(ImportJobContext context)
      throws ParseException {

    SqoopOptions opts = context.getOptions();
    Configuration conf = opts.getConf();

    String[] extraArgs = opts.getExtraArgs();


    conf.setBoolean(NETEZZA_DATASLICE_ALIGNED_ACCESS_OPT, false);

    if (extraArgs != null && extraArgs.length > 0
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

  @Override
  protected void propagateOptionsToJob(Job job) {
    super.propagateOptionsToJob(job);
    SqoopOptions opts = context.getOptions();
    Configuration conf = job.getConfiguration();
    conf.setIfUnset("pgbulkload.bin", "pg_bulkload");
    if (opts.getNullStringValue() != null) {
      conf.set("pgbulkload.null.string", opts.getNullStringValue());
    }
    setDelimiter("pgbulkload.input.field.delim",
                 opts.getInputFieldDelim(),
                 conf);
    setDelimiter("pgbulkload.input.record.delim",
                 opts.getInputRecordDelim(),
                 conf);
    setDelimiter("pgbulkload.input.enclosedby",
                 opts.getInputEnclosedBy(),
                 conf);
    setDelimiter("pgbulkload.input.escapedby",
                 opts.getInputEscapedBy(),
                 conf);
    conf.setBoolean("pgbulkload.input.encloserequired",
                    opts.isInputEncloseRequired());
    conf.setIfUnset("pgbulkload.check.constraints", "YES");
    conf.setIfUnset("pgbulkload.parse.errors", "INFINITE");
    conf.setIfUnset("pgbulkload.duplicate.errors", "INFINITE");
    conf.set("mapred.jar", context.getJarFile());
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
  public static final Log LOG = LogFactory.getLog(
      DefaultManagerFactory.class.getName());

  public ConnManager accept(JobData data) {
    SqoopOptions options = data.getSqoopOptions();

    String scheme = extractScheme(options);
    if (null == scheme) {
      // We don't know if this is a mysql://, hsql://, etc.
      // Can't do anything with this.
      LOG.warn("Null scheme associated with connect string.");
      return null;
    }

    LOG.debug("Trying with scheme: " + scheme);

    if (scheme.equals("jdbc:mysql:")) {
      if (options.isDirect()) {
        return new DirectMySQLManager(options);
      } else {
        return new MySQLManager(options);
      }
    } else if (scheme.equals("jdbc:postgresql:")) {
      if (options.isDirect()) {
        return new DirectPostgresqlManager(options);
      } else {
        return new PostgresqlManager(options);
      }
    } else if (scheme.startsWith("jdbc:hsqldb:")) {
      return new HsqldbManager(options);
    } else if (scheme.startsWith("jdbc:oracle:")) {
      return new OracleManager(options);
    } else if (scheme.startsWith("jdbc:sqlserver:")) {
      return new SQLServerManager(options);
    } else if (scheme.startsWith("jdbc:db2:")) {
      return new Db2Manager(options);
    } else if (scheme.startsWith("jdbc:netezza:")) {
      if (options.isDirect()) {
        return new DirectNetezzaManager(options);
      } else {
        return new NetezzaManager(options);
      }
    } else {
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
  }


  @Override
  public void runExport() throws ExportException, IOException {
    SqoopOptions options = context.getOptions();
    Configuration conf = options.getConf();
    DBConfiguration dbConf = null;
    String outputTableName = context.getTableName();
    String tableName = outputTableName;
    String tableClassName =
        new TableClassName(options).getClassForTable(outputTableName);
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
  }


  @Override
  protected int configureNumTasks(Job job) throws IOException {
    SqoopOptions options = context.getOptions();
    int numMapTasks = options.getNumMappers();
    if (numMapTasks < 1) {
      numMapTasks = SqoopOptions.DEFAULT_NUM_MAPPERS;
      LOG.warn("Invalid mapper count; using " + numMapTasks + " mappers.");
    }
View Full Code Here
183
184
185
186
187
188
189
  /**
   * @return a new SqoopOptions customized for this particular test, but one
   * which has not had any arguments parsed yet.
   */
  protected SqoopOptions getSqoopOptions(Configuration conf) {
    return new SqoopOptions(conf);
  }
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
230
      }

      manager = testServer.getManager();
    } else {
      Configuration conf = getConf();
      SqoopOptions opts = getSqoopOptions(conf);
      opts.setConnectString(getConnectString());
      opts.setTableName(getTableName());
      ConnFactory f = new ConnFactory(conf);
      try {
        this.manager = f.getManager(new JobData(opts, new ImportTool()));
      } catch (IOException ioe) {
        fail("IOException instantiating manager: "
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    expectExceptionInCharCode(254);
  }

  public void testDifferentTableNames() throws Exception {
    Configuration conf = new Configuration();
    SqoopOptions options = new SqoopOptions();
    TableDefWriter writer = new TableDefWriter(options, null,
        "inputTable", "outputTable", conf, false);

    Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
    writer.setColumnTypes(colTypes);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
    String targetDir = "targetDir";
    String inputTable = "inputTable";
    String outputTable = "outputTable";

    Configuration conf = new Configuration();
    SqoopOptions options = new SqoopOptions();
    // Specify a different target dir from input table name
    options.setTargetDir(targetDir);
    TableDefWriter writer = new TableDefWriter(options, null,
        inputTable, outputTable, conf, false);

    Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
    writer.setColumnTypes(colTypes);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
    String[] args = {
        "--hive-partition-key", "ds",
        "--hive-partition-value", "20110413",
    };
    Configuration conf = new Configuration();
    SqoopOptions options =
      new ImportTool().parseArguments(args, null, null, false);
    TableDefWriter writer = new TableDefWriter(options,
        null, "inputTable", "outputTable", conf, false);

    Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
View Full Code Here
TOP

Related Classes of com.cloudera.sqoop.SqoopOptions

  • com.cloudera.sqoop.hive.TestHiveImport
  • com.cloudera.sqoop.hive.TestTableDefWriter
  • com.cloudera.sqoop.lib.DelimiterSet
  • com.cloudera.sqoop.manager.CubridManagerExportTest
  • com.cloudera.sqoop.manager.CubridManagerImportTest
  • com.cloudera.sqoop.manager.DB2ManagerImportManualTest
  • com.cloudera.sqoop.manager.DirectMySQLExportTest
  • com.cloudera.sqoop.manager.DirectMySQLTest
  • com.cloudera.sqoop.manager.DirectNetezzaExportManualTest
  • com.cloudera.sqoop.manager.JdbcMySQLExportTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.