Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream.writeInt()

274
275
276
277
278
279
280
281
282
283
284
        // Write out length of each document (docnos are sequentially
        // ordered, so no need to explicitly keep track).
        int n = 0;
        for (int i = 1; i <= maxDocno - docnoOffset; i++) {
          out.writeInt(doclengths[i]);
          n++;
          context.getCounter(DocLengths.Count).increment(1);
          context.getCounter(DocLengths.SumOfDocLengths).increment(doclengths[i]);
        }
        LOG.info(n + " doc lengths written");
View Full Code Here
361
362
363
364
365
366
367
368
369
370
371
    //write the serialized task information to a file to pass to the truncater
    Path taskRanFilePath =
      ldirAlloc.getLocalPathForWrite(taskRanFile, conf);
    LocalFileSystem lfs = FileSystem.getLocal(conf);
    FSDataOutputStream out = lfs.create(taskRanFilePath);
    out.writeInt(allAttempts.size());
    for (Task t : allAttempts) {
      out.writeBoolean(t.isMapTask());
      t.write(out);
    }
    out.close();
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
      Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
      for (int i = 1000; i < 1050; i += 10) {
        Path recoveredEdits = new Path(
            recoveredEditsDir, String.format("%019d", i));
        FSDataOutputStream dos=  fs.create(recoveredEdits);
        dos.writeInt(i);
        dos.close();
      }
      long minSeqId = 2000;
      Path recoveredEdits = new Path(
          recoveredEditsDir, String.format("%019d", minSeqId-1));
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
      LOG.info("max docno: " + maxDocno);

      // Write out the doc length data into a single file.
      FSDataOutputStream out = fs.create(new Path(dataFile), true);

      out.writeInt(docnoOffset); // Write out the docno offset.
      out.writeInt(maxDocno - docnoOffset); // Write out the collection size.

      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
      // Write out the doc length data into a single file.
      FSDataOutputStream out = fs.create(new Path(dataFile), true);

      out.writeInt(docnoOffset); // Write out the docno offset.
      out.writeInt(maxDocno - docnoOffset); // Write out the collection size.

      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
      for (int i = 1; i <= maxDocno - docnoOffset; i++) {
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
      for (int i = 1; i <= maxDocno - docnoOffset; i++) {
        out.writeInt(doclengths[i]);
        n++;
        context.getCounter(DocLengths.Count).increment(1);
        context.getCounter(DocLengths.SumOfDocLengths).increment(doclengths[i]);
      }
      LOG.info(n + " doc lengths written");
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
        job.getCounters().write(dataOut);

        TaskCompletionEvent[] events =
                job.getTaskCompletionEvents(0, Integer.MAX_VALUE);
        dataOut.writeInt(events.length);
        for (TaskCompletionEvent event : events) {
          event.write(dataOut);
        }

        dataOut.close();
View Full Code Here
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
        // write the jobtracker.info file
        try {
          FSDataOutputStream out = FileSystem.create(fs, restartFile,
                                                     filePerm);
          out.writeInt(0);
          out.close();
        } catch (IOException ioe) {
          LOG.warn("Writing to file " + restartFile + " failed!");
          LOG.warn("FileSystem is not ready yet!");
          fs.delete(restartFile, false);
View Full Code Here
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
      //TODO This is similar to jobhistory recovery, maybe this common code
      //      can be factored out.
     
      // write to the tmp file
      FSDataOutputStream out = FileSystem.create(fs, tmpRestartFile, filePerm);
      out.writeInt(restartCount);
      out.close();

      // delete the main file
      fs.delete(restartFile, false);
     
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
    sLogger.info(cnt + " docs total. Done!");

    cnt = 0;
    sLogger.info("Writing " + outputFile);
    FSDataOutputStream out = fs.create(new Path(outputFile), true);
    out.writeInt(list.size());
    for (int i = 0; i < list.size(); i++) {
      out.writeUTF(list.get(i));
      cnt++;
      if (cnt % 100000 == 0) {
        sLogger.info(cnt + " docs");
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.