Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream.writeInt()

267
268
269
270
271
272
273
274
275
276
277
      LOG.info("max docno: " + maxDocno);

      // Write out the doc length data into a single file.
      FSDataOutputStream out = fs.create(new Path(dataFile), true);

      out.writeInt(docnoOffset); // Write out the docno offset.
      out.writeInt(maxDocno - docnoOffset); // Write out the collection size.

      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
      // Write out the doc length data into a single file.
      FSDataOutputStream out = fs.create(new Path(dataFile), true);

      out.writeInt(docnoOffset); // Write out the docno offset.
      out.writeInt(maxDocno - docnoOffset); // Write out the collection size.

      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
      for (int i = 1; i <= maxDocno - docnoOffset; i++) {
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
      // Write out length of each document (docnos are sequentially
      // ordered, so no need to explicitly keep track).
      int n = 0;
      for (int i = 1; i <= maxDocno - docnoOffset; i++) {
        out.writeInt(doclengths[i]);
        n++;
        context.getCounter(DocLengths.Count).increment(1);
        context.getCounter(DocLengths.SumOfDocLengths).increment(doclengths[i]);
      }
      LOG.info(n + " doc lengths written");
View Full Code Here
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
        // write the jobtracker.info file
        try {
          FSDataOutputStream out = FileSystem.create(fs, restartFile,
                                                     filePerm);
          out.writeInt(0);
          out.close();
        } catch (IOException ioe) {
          LOG.warn("Writing to file " + restartFile + " failed!");
          LOG.warn("FileSystem is not ready yet!");
          fs.delete(restartFile, false);
View Full Code Here
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
      //TODO This is similar to jobhistory recovery, maybe this common code
      //      can be factored out.
     
      // write to the tmp file
      FSDataOutputStream out = FileSystem.create(fs, tmpRestartFile, filePerm);
      out.writeInt(restartCount);
      out.close();

      // delete the main file
      fs.delete(restartFile, false);
     
View Full Code Here
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
        // disable recovery if this is a restart
        shouldRecover = false;

        // write the jobtracker.info file
        FSDataOutputStream out = FileSystem.create(fs, restartFile, filePerm);
        out.writeInt(0);
        out.close();
        return;
      }

      FSDataInputStream in = fs.open(restartFile);
View Full Code Here
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
      //TODO This is similar to jobhistory recovery, maybe this common code
      //      can be factored out.
     
      // write to the tmp file
      FSDataOutputStream out = FileSystem.create(fs, tmpRestartFile, filePerm);
      out.writeInt(restartCount);
      out.close();

      // delete the main file
      fs.delete(restartFile, false);
     
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
        job.getCounters().write(dataOut);

        TaskCompletionEvent[] events =
                job.getTaskCompletionEvents(0, Integer.MAX_VALUE);
        dataOut.writeInt(events.length);
        for (TaskCompletionEvent event : events) {
          event.write(dataOut);
        }

        dataOut.close();
View Full Code Here
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
        // write the jobtracker.info file
        try {
          FSDataOutputStream out = FileSystem.create(fs, restartFile,
                                                     filePerm);
          out.writeInt(0);
          out.close();
        } catch (IOException ioe) {
          LOG.warn("Writing to file " + restartFile + " failed!");
          LOG.warn("FileSystem is not ready yet!");
          fs.delete(restartFile, false);
View Full Code Here
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
      //TODO This is similar to jobhistory recovery, maybe this common code
      //      can be factored out.
     
      // write to the tmp file
      FSDataOutputStream out = FileSystem.create(fs, tmpRestartFile, filePerm);
      out.writeInt(restartCount);
      out.close();

      // delete the main file
      fs.delete(restartFile, false);
     
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.