Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.DataOutputBuffer.reset()

630
631
632
633
634
635
636
637
638
639
640
    WritableUtils.writeVInt(out, splits.length);
    DataOutputBuffer buffer = new DataOutputBuffer();
    RawSplit rawSplit = new RawSplit();
    for(InputSplit split: splits) {
      rawSplit.setClassName(split.getClass().getName());
      buffer.reset();
      split.write(buffer);
      rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
      rawSplit.setLocations(split.getLocations());
      rawSplit.write(out);
    }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
        }
        DataOutputBuffer buffer = new DataOutputBuffer();
        for (int i = 0; i < splits.length; i++) {
          String mapId = "map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId,
                                    mapId, i,
View Full Code Here
463
464
465
466
467
468
469
470
471
472
473
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(),
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);
        writer.append(key, value);
        reporter.progress();
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
          SerializationFactory factory = new SerializationFactory(conf);
          Serializer serializer =
            factory.getSerializer(splits.get(0).getClass());
          serializer.open(buffer);
          for (int i = 0; i < splits.size(); i++) {
            buffer.reset();
            serializer.serialize(splits.get(i));
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits.get(i).getClass().getName());
            rawSplit.setDataLength(splits.get(i).getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
        } else {
          InputSplit[] splits = job.getInputFormat().getSplits(job, 1);
          rawSplits = new RawSplit[splits.length];
          DataOutputBuffer buffer = new DataOutputBuffer();
          for (int i = 0; i < splits.length; i++) {
            buffer.reset();
            splits[i].write(buffer);
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits[i].getClass().getName());
            rawSplit.setDataLength(splits[i].getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
View Full Code Here
834
835
836
837
838
839
840
841
842
843
844
    try {
      DataOutputBuffer buffer = new DataOutputBuffer();
      RawSplit rawSplit = new RawSplit();
      for(InputSplit split: splits) {
        rawSplit.setClassName(split.getClass().getName());
        buffer.reset();
        split.write(buffer);
        rawSplit.setDataLength(split.getLength());
        rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
        rawSplit.setLocations(split.getLocations());
        rawSplit.write(out);
View Full Code Here
899
900
901
902
903
904
905
906
907
908
909
        Serializer<T> serializer =
          factory.getSerializer((Class<T>) array[0].getClass());
        serializer.open(buffer);
        for(T split: array) {
          rawSplit.setClassName(split.getClass().getName());
          buffer.reset();
          serializer.serialize(split);
          rawSplit.setDataLength(split.getLength());
          rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
          rawSplit.setLocations(split.getLocations());
          rawSplit.write(out);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
      outQw.write(buffer);

      in.reset(buffer.getData(), 0, buffer.getLength());
      inQw.readFields(in);

      buffer.reset();
    }
    long e = System.nanoTime();
    System.out.println((e - s) / 1000000.0 / (double) count);
    // System.out.println((e - s) / (double) count);
  }
View Full Code Here
662
663
664
665
666
667
668
669
670
671
672
    WritableUtils.writeVInt(out, splits.length);
    DataOutputBuffer buffer = new DataOutputBuffer();
    RawSplit rawSplit = new RawSplit();
    for(InputSplit split: splits) {
      rawSplit.setClassName(split.getClass().getName());
      buffer.reset();
      split.write(buffer);
      rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
      rawSplit.setLocations(split.getLocations());
      rawSplit.write(out);
    }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
        DataOutputBuffer buffer = new DataOutputBuffer();
        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId,
                                    mapId, i,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.