Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.DataOutputBuffer


        = new IdxIndexDescriptor(qualifierName2, IdxQualifierType.LONG);

    descriptor.addIndexDescriptor(indexDescriptor1);
    descriptor.addIndexDescriptor(indexDescriptor2);

    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    descriptor.write(dataOutputBuffer);

    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());

    IdxColumnDescriptor clonedDescriptor = new IdxColumnDescriptor();
    clonedDescriptor.readFields(dataInputBuffer);

    Assert.assertEquals("The expression was not the same after being written and read", descriptor, clonedDescriptor);
View Full Code Here


  public void testWritable() throws IOException {
    byte[] qualifierName1 = Bytes.toBytes("qualifer1");
    IdxIndexDescriptor descriptor
        = new IdxIndexDescriptor(qualifierName1, IdxQualifierType.INT);

    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    descriptor.write(dataOutputBuffer);

    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());

    IdxIndexDescriptor clonedDescriptor = new IdxIndexDescriptor();
    clonedDescriptor.readFields(dataInputBuffer);

    Assert.assertEquals("The descriptor was not the same after being written and read", descriptor, clonedDescriptor);
View Full Code Here

   * @throws java.io.IOException if an io error occurs
   */
  public void testWritable() throws IOException {
    Expression expression = Expression.comparison("columnName1", "qualifier1", Comparison.Operator.EQ, Bytes.toBytes("value"));

    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    expression.write(dataOutputBuffer);

    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());

    Expression clonedExpression = new Comparison();
    clonedExpression.readFields(dataInputBuffer);

    Assert.assertEquals("The expression was not the same after being written and read", expression, clonedExpression);
View Full Code Here

            Expression.comparison("columnName2", "qualifier2", Comparison.Operator.GT, Bytes.toBytes("value2")),
            Expression.comparison("columnName3", "qualifier3", Comparison.Operator.LT, Bytes.toBytes("value3"))
        )
    );

    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    expression.write(dataOutputBuffer);

    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());

    Expression clonedExpression = new Or();
    clonedExpression.readFields(dataInputBuffer);

    Assert.assertEquals("The expression was not the same after being written and read", expression, clonedExpression);
View Full Code Here

      key = (WritableComparable)ReflectionUtils.newInstance(
          r.getKeyClass(), getConf());
      val = (Writable)ReflectionUtils.newInstance(
          r.getValueClass(), getConf());
      inbuf = new DataInputBuffer();
      outbuf = new DataOutputBuffer();
    }
View Full Code Here

      this.partitions = job.getNumReduceTasks();
      this.partitioner = (Partitioner)ReflectionUtils.newInstance(
                                                                  job.getPartitionerClass(), job);
      maxBufferSize = job.getInt("io.sort.mb", 100) * 1024 * 1024 / 2;
      this.sortSpillException = null;
      keyValBuffer = new DataOutputBuffer();

      this.job = job;
      this.reporter = reporter;
      this.comparator = job.getOutputKeyComparator();
      this.keyClass = job.getMapOutputKeyClass();
View Full Code Here

      if (sortSpillException != null) {
        throw (IOException) new IOException().initCause(sortSpillException);
      }
     
      if (keyValBuffer == null) {
        keyValBuffer = new DataOutputBuffer();
        sortImpl = new BufferSorter[partitions];
        for (int i = 0; i < partitions; i++)
          sortImpl[i] = (BufferSorter)ReflectionUtils.newInstance(
                            job.getClass("map.sort.class", MergeSorter.class,
                                         BufferSorter.class), job);
View Full Code Here

        throw new RuntimeException(e);
      }

      DataInputBuffer keyIn = new DataInputBuffer();
      DataInputBuffer valIn = new DataInputBuffer();
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(),
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);
        writer.append(key, value);
        reporter.progress();
      }
    }
View Full Code Here

   */
  private void writeSplitsFile(InputSplit[] splits, FSDataOutputStream out) throws IOException {
    out.write(SPLIT_FILE_HEADER);
    WritableUtils.writeVInt(out, CURRENT_SPLIT_FILE_VERSION);
    WritableUtils.writeVInt(out, splits.length);
    DataOutputBuffer buffer = new DataOutputBuffer();
    RawSplit rawSplit = new RawSplit();
    for(InputSplit split: splits) {
      rawSplit.setClassName(split.getClass().getName());
      buffer.reset();
      split.write(buffer);
      rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
      rawSplit.setLocations(split.getLocations());
      rawSplit.write(out);
    }
  }
View Full Code Here

          if (!outputFs.mkdirs(tmpDir)) {
            LOG.error("Mkdirs failed to create " + tmpDir.toString());
          }
        }

        DataOutputBuffer buffer = new DataOutputBuffer();
        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId,
                                    mapId, i,
                                    splits[i].getClass().getName(),
                                    split);
          JobConf localConf = new JobConf(job);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.DataOutputBuffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.