Package org.apache.sqoop.job.etl

Examples of org.apache.sqoop.job.etl.PartitionerContext

77
78
79
80
81
82
83
84
85
86
    HdfsExportPartitioner partitioner = new HdfsExportPartitioner();
    PrefixContext prefixContext = new PrefixContext(conf, "");
    int[] partitionValues = {2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 17};

    for(int maxPartitions : partitionValues) {
      PartitionerContext partCont = new PartitionerContext(prefixContext, maxPartitions, null);
      List<Partition> partitionList = partitioner.getPartitions(partCont, null, null);
      assertTrue(partitionList.size()<=maxPartitions);
    }
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    Object connectorConnection = ConfigurationUtils.getConfigConnectorConnection(conf);
    Object connectorJob = ConfigurationUtils.getConfigConnectorJob(conf);
    Schema schema = ConfigurationUtils.getConnectorSchema(conf);

    long maxPartitions = conf.getLong(JobConstants.JOB_ETL_EXTRACTOR_NUM, 10);
    PartitionerContext partitionerContext = new PartitionerContext(connectorContext, maxPartitions, schema);

    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connectorConnection, connectorJob);
    List<InputSplit> splits = new LinkedList<InputSplit>();
    for (Partition partition : partitions) {
      LOG.debug("Partition: " + partition);
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -3",
        "-3 <= ICOL AND ICOL < -1",
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -1",
        "-1 <= ICOL AND ICOL < 2",
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 13, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -4",
        "-4 <= ICOL AND ICOL < -3",
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5.0 <= DCOL AND DCOL < -3.0",
        "-3.0 <= DCOL AND DCOL < -1.0",
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5.0 <= DCOL AND DCOL < -1.6666666666666665",
        "-1.6666666666666665 <= DCOL AND DCOL < 1.666666666666667",
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -3",
        "-3 <= ICOL AND ICOL < -1",
View Full Code Here
230
231
232
233
234
235
236
237
238
239
240
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[]{
      "-5 <= DCOL AND DCOL < -2",
      "-2 <= DCOL AND DCOL < 1",
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();

    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);

    verifyResult(partitions, new String[]{
      "DCOL = -5",
    });
View Full Code Here
TOP

Related Classes of org.apache.sqoop.job.etl.PartitionerContext

  • org.apache.sqoop.connector.jdbc.TestImportPartitioner
  • org.apache.sqoop.job.mr.SqoopInputFormat
  • org.apache.sqoop.job.TestHdfsExtract
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.