Package org.apache.sqoop.job.io

Examples of org.apache.sqoop.job.io.Data

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
  }

  private void verifyOutputText(BufferedReader reader) throws IOException {
    String actual = null;
    String expected;
    Data data = new Data();
    int index = START_ID*NUMBER_OF_ROWS_PER_ID;
    while ((actual = reader.readLine()) != null){
      data.setContent(new Object[] {
        index, (double) index, String.valueOf(index) },
          Data.ARRAY_RECORD);
      expected = data.toString();
      index++;

      assertEquals(expected, actual);
    }
    reader.close();
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
  private void verifyOutputSequence(SequenceFile.Reader reader) throws IOException {
    int index = START_ID*NUMBER_OF_ROWS_PER_ID;
    Text actual = new Text();
    Text expected = new Text();
    Data data = new Data();
    while (reader.next(actual)){
      data.setContent(new Object[] {
          index, (double) index, String.valueOf(index) },
          Data.ARRAY_RECORD);
      expected.set(data.toString());
      index++;

      assertEquals(expected.toString(), actual.toString());
    }
    reader.close();
View Full Code Here
118
119
120
121
122
123
124
125
126
127
    }

    @Override
    public void setFieldDelimiter(char fieldDelimiter) {
      if (data == null) {
        data = new Data();
      }

      data.setFieldDelimiter(fieldDelimiter);
    }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    }

    @Override
    public void writeContent(Object content, int type) {
      if (data == null) {
        data = new Data();
      }

      data.setContent(content, type);
      try {
        context.write(data, NullWritable.get());
View Full Code Here
58
59
60
61
62
63
64
65
  private String loaderName;

  SqoopOutputFormatLoadExecutor(boolean isTest, String loaderName){
    this.isTest = isTest;
    this.loaderName = loaderName;
    data = new Data();
    producer = new SqoopRecordWriter();
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
    data = new Data();
    producer = new SqoopRecordWriter();
  }

  public SqoopOutputFormatLoadExecutor(JobContext jobctx) {
    data = new Data();
    context = jobctx;
    producer = new SqoopRecordWriter();
  }
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    ConfigurationUtils.setJobType(conf, MJob.Type.EXPORT);
    conf.set(JobConstants.JOB_ETL_LOADER, ThrowingLoader.class.getName());
    SqoopOutputFormatLoadExecutor executor = new
        SqoopOutputFormatLoadExecutor(true, ThrowingLoader.class.getName());
    RecordWriter<Data, NullWritable> writer = executor.getRecordWriter();
    Data data = new Data();
    try {
      for (int count = 0; count < 100; count++) {
        data.setContent(String.valueOf(count), Data.CSV_RECORD);
        writer.write(data, null);
      }
    } catch (SqoopException ex) {
      throw ex.getCause();
    }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
    ConfigurationUtils.setJobType(conf, MJob.Type.EXPORT);
    conf.set(JobConstants.JOB_ETL_LOADER, GoodContinuousLoader.class.getName());
    SqoopOutputFormatLoadExecutor executor = new
        SqoopOutputFormatLoadExecutor(true, GoodContinuousLoader.class.getName());
    RecordWriter<Data, NullWritable> writer = executor.getRecordWriter();
    Data data = new Data();
    for (int i = 0; i < 10; i++) {
      StringBuilder builder = new StringBuilder();
      for (int count = 0; count < 100; count++) {
        builder.append(String.valueOf(count));
        if (count != 99) {
          builder.append(",");
        }
      }
      data.setContent(builder.toString(), Data.CSV_RECORD);
      writer.write(data, null);
    }
    writer.close(null);
  }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
  @Test (expected = SqoopException.class)
  public void testSuccessfulLoader() throws Throwable {
    SqoopOutputFormatLoadExecutor executor = new
        SqoopOutputFormatLoadExecutor(true, GoodLoader.class.getName());
    RecordWriter<Data, NullWritable> writer = executor.getRecordWriter();
    Data data = new Data();
    StringBuilder builder = new StringBuilder();
    for (int count = 0; count < 100; count++) {
      builder.append(String.valueOf(count));
      if (count != 99) {
        builder.append(",");
      }
    }
    data.setContent(builder.toString(), Data.CSV_RECORD);
    writer.write(data, null);
    //Allow writer to complete.
    TimeUnit.SECONDS.sleep(5);
    writer.close(null);
  }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    ConfigurationUtils.setJobType(conf, MJob.Type.EXPORT);
    conf.set(JobConstants.JOB_ETL_LOADER, ThrowingContinuousLoader.class.getName());
    SqoopOutputFormatLoadExecutor executor = new
        SqoopOutputFormatLoadExecutor(true, ThrowingContinuousLoader.class.getName());
    RecordWriter<Data, NullWritable> writer = executor.getRecordWriter();
    Data data = new Data();
    try {
      for (int i = 0; i < 10; i++) {
        StringBuilder builder = new StringBuilder();
        for (int count = 0; count < 100; count++) {
          builder.append(String.valueOf(count));
          if (count != 99) {
            builder.append(",");
          }
        }
        data.setContent(builder.toString(), Data.CSV_RECORD);
        writer.write(data, null);
      }
      writer.close(null);
    } catch (SqoopException ex) {
      throw ex.getCause();
View Full Code Here
TOP

Related Classes of org.apache.sqoop.job.io.Data

  • org.apache.sqoop.job.mr.SqoopMapper$MapDataWriter
  • org.apache.sqoop.job.mr.SqoopOutputFormatLoadExecutor
  • org.apache.sqoop.job.mr.TestSqoopOutputFormatLoadExecutor
  • org.apache.sqoop.job.TestHdfsLoad
  • java.io.IOException
  • org.apache.sqoop.common.SqoopException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.