Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream.writeInt()

126
127
128
129
130
131
132
133
134
135
136
    os.write(bytes);

    // Serialize the ints.
    os.writeInt(intList.size());
    for (int i = 0; i < intList.size(); i++) {
      os.writeInt(intList.getInt(i));
    }
   
    os.close();
  }
}
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
    LOG.info("Done!");

    LOG.info("Writing " + output);
    FSDataOutputStream out = fs.create(output, true);
    reader = new LineReader(fs.open(input));
    out.writeInt(cnt);
    cnt = 0;
    while (reader.readLine(line) > 0) {
      String[] arr = line.toString().split("\\t");
      out.writeUTF(arr[0]);
      cnt++;
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
    LineReader reader = new LineReader(fs.open(new Path(outputPath + "/part-00000")));
    FSDataOutputStream out = fs.create(new Path(indexFile), true);

    out.writeUTF(ClueWarcForwardIndex.class.getCanonicalName());
    out.writeUTF(collectionPath);
    out.writeInt(blocks);

    int cnt = 0;
    Text line = new Text();
    while (reader.readLine(line) > 0) {
      String[] arr = line.toString().split("\\s+");
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
      int docno = Integer.parseInt(arr[0]);
      int offset = Integer.parseInt(arr[1]);
      short fileno = Short.parseShort(arr[2]);

      out.writeInt(docno);
      out.writeInt(offset);
      out.writeShort(fileno);

      cnt++;
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
      int docno = Integer.parseInt(arr[0]);
      int offset = Integer.parseInt(arr[1]);
      short fileno = Short.parseShort(arr[2]);

      out.writeInt(docno);
      out.writeInt(offset);
      out.writeShort(fileno);

      cnt++;

      if (cnt % 100000 == 0) {
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    LOG.info("Done! Total of " + cnt + " docids read.");

    cnt = 0;
    LOG.info("Writing " + output);
    FSDataOutputStream out = fs.create(output, true);
    out.writeInt(list.size());
    for (int i = 0; i < list.size(); i++) {
      out.writeInt(list.get(i));
      cnt++;
      if (cnt % 500000 == 0) {
        LOG.info(cnt);
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
    cnt = 0;
    LOG.info("Writing " + output);
    FSDataOutputStream out = fs.create(output, true);
    out.writeInt(list.size());
    for (int i = 0; i < list.size(); i++) {
      out.writeInt(list.get(i));
      cnt++;
      if (cnt % 500000 == 0) {
        LOG.info(cnt);
      }
    }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    LineReader reader = new LineReader(fs.open(new Path(tmpPath + "/part-00000")));
    FSDataOutputStream out = fs.create(new Path(indexFile), true);

    out.writeUTF(edu.umd.cloud9.collection.wikipedia.WikipediaForwardIndex.class.getCanonicalName());
    out.writeUTF(inputPath.toString());
    out.writeInt(blocks);

    int cnt = 0;
    Text line = new Text();
    while (reader.readLine(line) > 0) {
      String[] arr = line.toString().split("\\s+");
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
      int docno = Integer.parseInt(arr[0]);
      int offset = Integer.parseInt(arr[1]);
      short fileno = Short.parseShort(arr[2]);

      out.writeInt(docno);
      out.writeInt(offset);
      out.writeShort(fileno);

      cnt++;
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
      int docno = Integer.parseInt(arr[0]);
      int offset = Integer.parseInt(arr[1]);
      short fileno = Short.parseShort(arr[2]);

      out.writeInt(docno);
      out.writeInt(offset);
      out.writeShort(fileno);

      cnt++;

      if (cnt % 100000 == 0) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.