Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.IntWritable

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    Path path = new Path(outputPath);
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    VectorWritable vw = new VectorWritable();
    IntWritable iw = new IntWritable();
    for(int i=0; i<eigenVectors.numRows() - 1; i++) {
      Vector v = eigenVectors.getRow(i);
      v.setName("eigenVector" + i + ", eigenvalue = " + eigenValues.get(i));
      vw.set(v);
      iw.set(i);
      seqWriter.append(iw, vw);
    }
    seqWriter.close();
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
    Vector vector = value.get();
    Iterator<Element> it = vector.iterateNonZero();
   
    while (it.hasNext()) {
      Element e = it.next();
      output.collect(new IntWritable(e.index()), ONE);
    }
    output.collect(TOTAL_COUNT, ONE);
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      }
      Path dictionaryFile = new Path(localFiles[0].getPath());
      FileSystem fs = dictionaryFile.getFileSystem(job);
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, dictionaryFile, job);
      Text key = new Text();
      IntWritable value = new IntWritable();
     
      // key is word value is id
      while (reader.next(key, value)) {
        dictionary.put(key.toString(), value.get());
      }
    } catch (IOException e) {
      throw new IllegalStateException(e);
    }
  }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        InMemInputSplit split = (InMemInputSplit) splits[index];
        InMemRecordReader reader = (InMemRecordReader) inputFormat.getRecordReader(
            split, conf, null);

        for (int tree = 0; tree < split.getNbTrees(); tree++) {
          IntWritable key = reader.createKey();
          NullWritable value = reader.createValue();

          // reader.next() should return true until there is no tree left
          assertEquals(tree < split.getNbTrees(), reader.next(key, value));
         
          assertEquals(split.getFirstId() + tree, key.get());
        }
      }
    }
  }
View Full Code Here
227
228
229
230
231
232
233
234
235
236
237
        }
       
        int fieldSize = DICTIONARY_BYTE_OVERHEAD + key.toString().length() * 2
                        + Integer.SIZE / 8;
        currentChunkSize += fieldSize;
        dictWriter.append(key, new IntWritable(i++));
      }
    }
    maxTermDimension[0] = i;
    dictWriter.close();
   
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    Iterator<Vector.Element> it = userVector.get().iterateNonZero();
    while (it.hasNext()) {
      Vector.Element next1 = it.next();
      int index1 = next1.index();
      Iterator<Vector.Element> it2 = userVector.get().iterateNonZero();
      IntWritable itemWritable1 = new IntWritable(index1);
      while (it2.hasNext()) {
        Vector.Element next2 = it2.next();
        int index2 = next2.index();
        if (index1 != index2) {
          output.collect(itemWritable1, new IntWritable(index2));
        }
      }
    }
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
      Vector.Element element = userVectorIterator.next();
      int index = element.index();
      double value = element.get();
      Vector columnVector;
      try {
        columnVector = cooccurrenceColumnCache.get(new IntWritable(index));
      } catch (TasteException te) {
        if (te.getCause() instanceof IOException) {
          throw (IOException) te.getCause();
        } else {
          throw new IOException(te.getCause());
        }
      }
      columnVector.times(value).addTo(recommendationVector);
    }
   
    Queue<RecommendedItem> topItems = new PriorityQueue<RecommendedItem>(recommendationsPerUser + 1,
        Collections.reverseOrder());
   
    Iterator<Vector.Element> recommendationVectorIterator = recommendationVector.iterateNonZero();
    LongWritable itemID = new LongWritable();
    while (recommendationVectorIterator.hasNext()) {
      Vector.Element element = recommendationVectorIterator.next();
      int index = element.index();
      if (userVector.get(index) == 0.0) {
        if (topItems.size() < recommendationsPerUser) {
          indexItemIDMap.get(new IntWritable(index), itemID);
          topItems.add(new GenericRecommendedItem(itemID.get(), (float) element.get()));
        } else if (element.get() > topItems.peek().getValue()) {
          indexItemIDMap.get(new IntWritable(index), itemID);
          topItems.add(new GenericRecommendedItem(itemID.get(), (float) element.get()));
          topItems.poll();
        }
      }
    }
View Full Code Here
39
40
41
42
43
44
45
                  OutputCollector<IntWritable,LongWritable> output,
                  Reporter reporter) throws IOException {
    String[] tokens = ItemIDIndexMapper.COMMA.split(value.toString());
    long itemID = Long.parseLong(tokens[1]);
    int index = idToIndex(itemID);
    output.collect(new IntWritable(index), new LongWritable(itemID));
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
   */
  public static String[] loadTermDictionary(Configuration conf, FileSystem fs, String filePattern) throws IOException {
    FileStatus[] dictionaryFiles = fs.globStatus(new Path(filePattern));
    OpenObjectIntHashMap<String> dict = new OpenObjectIntHashMap<String>();
    Text key = new Text();
    IntWritable value = new IntWritable();
    for (FileStatus fileStatus : dictionaryFiles) {
      Path path = fileStatus.getPath();
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
      // key is term value is id
      while (reader.next(key, value)) {
        dict.put(key.toString(), value.get());
      }
    }
    String[] dictionary = new String[dict.size()];
    for (String feature : dict.keys()) {
      dictionary[dict.get(feature)] = feature;
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    SequenceFile.Writer matrixWriter = SequenceFile.createWriter(fs,
                                                                 conf,
                                                                 matrixPath,
                                                                 IntWritable.class,
                                                                 VectorWritable.class);
    IntWritable docId = new IntWritable();
    Text inputKey = new Text();
    VectorWritable v = new VectorWritable();

    int i = 0;
    for(FileStatus status : fs.listStatus(inputPath)) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, status.getPath(), conf);
      while(reader.next(inputKey, v)) {
        docId.set(i);
        indexWriter.append(docId, inputKey);
        matrixWriter.append(docId, v);
        i++;
      }
      reader.close();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.IntWritable

  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.aggregation.TemperatureAveragingPair
  • bbejeck.mapred.coocurrance.PairsRelativeOccurrenceReducerTest
  • brickhouse.udf.collect.CollectMaxTest
  • co.nubetech.hiho.dedup.TestDedupJob
  • co.nubetech.hiho.merge.TestMergeJob
  • co.nubetech.hiho.similarity.ngram.TestScoreMapper
  • co.nubetech.hiho.similarity.ngram.TestScoreReducer
  • com.alimama.mdrill.index.IndexReducer
  • com.datasalt.pangool.utils.HadoopUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.