Package com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat

Examples of com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat.TupleInputReader

164
165
166
167
168
169
170
171
172
173
174
175
176
 
  /*
   * Read the Tuples from a TupleOutput using TupleInputReader.
   */
  public static void readTuples(Path file, Configuration conf, TupleVisitor iterator) throws IOException, InterruptedException {
    TupleInputReader reader = new TupleInputReader(conf);
    reader.initialize(new Path(file + ""), conf);
    while(reader.nextKeyValueNoSync()) {
      ITuple tuple = reader.getCurrentKey();
      iterator.onTuple(tuple);
    }
    reader.close();
  }
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
33
    if(args.length == 2 && args[1].equals("local")) {
      fS = FileSystem.getLocal(conf);
    }
   
    for(FileStatus fStatus: fS.globStatus(path)) {
      TupleInputReader reader = new TupleInputReader(conf);
      reader.initialize(fStatus.getPath(), conf);
      while(reader.nextKeyValueNoSync()) {
        System.out.println(fStatus.getPath() + "\t" + reader.getCurrentKey());
      }
    }
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    }
    // Use a HashSet to calculate the total vocabulary size
    Set<String> vocabulary = new HashSet<String>();
    // Read tuples from generate job
    for(FileStatus fileStatus : fileSystem.globStatus(generatedModel)) {
      TupleInputReader reader = new TupleInputReader(conf);
      reader.initialize(fileStatus.getPath(), conf);
      while(reader.nextKeyValueNoSync()) {
        // Read Tuple
        ITuple tuple = reader.getCurrentKey();
        Integer count = (Integer) tuple.get("count");
        Category category = (Category) tuple.get("category");
        String word = tuple.get("word").toString();
        vocabulary.add(word);
        tokensPerCategory.put(category, MapUtils.getInteger(tokensPerCategory, category, 0) + count);
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
  public List<Pair<Object, Object>> ensureTupleOutput(String output) throws IOException {
    List<Pair<Object, Object>> outs = outputs.get(output);
    try {
      if(outs == null) {
        outs = new ArrayList<Pair<Object, Object>>();
        TupleInputReader reader = new TupleInputReader(getConf());
        reader.initialize(new Path(output), getConf());
        while(reader.nextKeyValueNoSync()) {
          ITuple tuple = reader.getCurrentKey();
          outs.add(new Pair<Object, Object>(tuple, NullWritable.get()));
        }
        reader.close();
      }
    } catch(InterruptedException e) {
      throw new IOException(e);
    }
    return outs;
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    }
    // Use a HashSet to calculate the total vocabulary size
    Set<String> vocabulary = new HashSet<String>();
    // Read tuples from generate job
    for(FileStatus fileStatus : fileSystem.globStatus(generatedModel)) {
      TupleInputReader reader = new TupleInputReader(conf);
      reader.initialize(fileStatus.getPath(), conf);
      while(reader.nextKeyValueNoSync()) {
        // Read Tuple
        ITuple tuple = reader.getCurrentKey();
        Integer count = (Integer) tuple.get("count");
        Category category = (Category) tuple.get("category");
        String word = tuple.get("word").toString();
        vocabulary.add(word);
        tokensPerCategory.put(category, MapUtils.getInteger(tokensPerCategory, category, 0) + count);
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat.TupleInputReader

  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.TupleViewer
  • com.datasalt.pangool.utils.test.AbstractHadoopTestLibrary
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.