Package org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat

Examples of org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
  @Override
  public void setup(Context context) {
    Configuration conf = context.getConfiguration();
   
    WikipediaInputSplit wiSplit = (WikipediaInputSplit)context.getInputSplit();
    myGroup = wiSplit.getPartition();
    numGroups = WikipediaConfiguration.getNumGroups(conf);
   
    FileSplit split = wiSplit.getFileSplit();
    String fileName = split.getPath().getName();
    Matcher matcher = languagePattern.matcher(fileName);
    if (matcher.matches()) {
      language = matcher.group(1).replace('_', '-').toLowerCase();
    } else {
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    tablename = new Text(WikipediaConfiguration.getTableName(conf));
    indexTableName = new Text(tablename + "Index");
    reverseIndexTableName = new Text(tablename + "ReverseIndex");
    metadataTableName = new Text(tablename + "Metadata");
   
    WikipediaInputSplit wiSplit = (WikipediaInputSplit)context.getInputSplit();
    myGroup = wiSplit.getPartition();
    numGroups = WikipediaConfiguration.getNumGroups(conf);
   
    FileSplit split = wiSplit.getFileSplit();
    String fileName = split.getPath().getName();
    Matcher matcher = languagePattern.matcher(fileName);
    if (matcher.matches()) {
      language = matcher.group(1).replace('_', '-').toLowerCase();
    } else {
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
  public void testIncorrectArgs() throws Exception {
    File f = createFile(xml1);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
    AggregatingRecordReader reader = new AggregatingRecordReader();
    try {
      // Clear the values for BEGIN and STOP TOKEN
      conf.set(AggregatingRecordReader.START_TOKEN, null);
      conf.set(AggregatingRecordReader.END_TOKEN, null);
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
  public void testCorrectXML() throws Exception {
    File f = createFile(xml1);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
  public void testPartialXML() throws Exception {
    File f = createFile(xml2);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
    conf.set(AggregatingRecordReader.RETURN_PARTIAL_MATCHES, Boolean.toString(false));
    File f = createFile(xml3);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
  public void testPartialXML2() throws Exception {
    File f = createFile(xml3);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
  public void testLineSplitting() throws Exception {
    File f = createFile(xml4);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
  @Test
  public void testNoEndTokenHandling() throws Exception {
    File f = createFile(xml5);
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue("Not enough records returned.", reader.nextKeyValue());
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    Assert.assertNotNull(url);
    File data = new File(url.toURI());
    Path tmpFile = new Path(data.getAbsolutePath());
   
    // Setup the Mapper
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null),0);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
View Full Code Here
TOP

Related Classes of org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit

  • org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputSplitTest
  • org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper
  • org.apache.accumulo.examples.wikisearch.ingest.WikipediaPartitioner
  • org.apache.accumulo.examples.wikisearch.logic.TestQueryLogic
  • org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReaderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.