Examples of org.apache.hadoop.io.Writable

org.apache.hadoop.io.Writable
A simple, efficient, serialization protocol, based on {@link DataInput} and{@link DataOutput}.
Implementations typically implement a static read(DataInput) method which constructs a new instance, calls {@link #readFields(DataInput)}, and returns the instance.

    
    List<Cluster> clusters = new ArrayList<Cluster>();
    
    SequenceFile.Reader reader = new SequenceFile.Reader(
        fs, path, conf);
    Writable key = (Writable) reader.getKeyClass()
        .newInstance();
    Writable value = (Writable) reader.getValueClass()
        .newInstance();
    
    while (reader.next(key, value)) {
      Cluster cluster = (Cluster) value;
      clusters.add(cluster);

View Full Code Here


    // read the whole file
    org.apache.hadoop.mapred.RecordReader reader =
        in.getRecordReader(splits[0], conf, Reporter.NULL);
    Object key = reader.createKey();
    Writable value = (Writable) reader.createValue();
    int rowNum = 0;
    List<? extends StructField> fields =inspector.getAllStructFieldRefs();
    IntObjectInspector intInspector =
        (IntObjectInspector) fields.get(0).getFieldObjectInspector();
    assertEquals(0.0, reader.getProgress(), 0.00001);

View Full Code Here


    // read the whole file
    org.apache.hadoop.mapred.RecordReader reader =
        in.getRecordReader(splits[0], conf, Reporter.NULL);
    Object key = reader.createKey();
    Writable value = (Writable) reader.createValue();
    List<? extends StructField> fields =inspector.getAllStructFieldRefs();
    StringObjectInspector strInspector = (StringObjectInspector)
        fields.get(0).getFieldObjectInspector();
    assertEquals(true, reader.next(key, value));
    assertEquals("owen", strInspector.getPrimitiveJavaObject(inspector.

View Full Code Here

  /** Called for each call. */
  public abstract Writable call(Writable param) throws IOException;


  
  private Writable makeParam() {
    Writable param;                               // construct param
    try {
      param = (Writable)paramClass.newInstance();
      if (param instanceof Configurable) {
        ((Configurable)param).setConf(conf);
      }

View Full Code Here

          }
        
          if (LOG.isLoggable(Level.FINE))
            LOG.fine(getName() + " got #" + id);
        
          Writable param = makeParam();           // read param
          param.readFields(in);        
        
          Call call = new Call(id, param, this);
        
          synchronized (callQueue) {
            callQueue.addLast(call);              // queue the call

View Full Code Here

          if (LOG.isLoggable(Level.FINE))
            LOG.fine(getName() + ": has #" + call.id + " from " +
                     call.connection.socket.getInetAddress().getHostAddress());
          
          String error = null;
          Writable value = null;
          try {
            value = call(call.param);             // make the call
          } catch (IOException e) {
            LOG.log(Level.INFO, getName() + " call error: " + e, e);
            error = getStackTrace(e);
          } catch (Exception e) {
            LOG.log(Level.INFO, getName() + " call error: " + e, e);
            error = getStackTrace(e);
          }
            
          DataOutputStream out = call.connection.out;
          synchronized (out) {
            out.writeInt(call.id);                // write call id
            out.writeBoolean(error!=null);        // write error flag
            if (error != null)
              value = new UTF8(error);
            value.write(out);                     // write value
            out.flush();
          }


        } catch (Exception e) {
          LOG.log(Level.INFO, getName() + " caught: " + e, e);

View Full Code Here

    }
    return connection;
  }


  private Writable makeValue() {
    Writable value;                             // construct value
    try {
      value = (Writable)valueClass.newInstance();
    } catch (InstantiationException e) {
      throw new RuntimeException(e.toString());
    } catch (IllegalAccessException e) {

View Full Code Here

          if (isError) {
            UTF8 utf8 = new UTF8();
            utf8.readFields(in);                  // read error string
            call.setResult(null, utf8.toString());
          } else {
            Writable value = makeValue();
            try {
              readingCall = call;
              if(value instanceof Configurable) {
                ((Configurable) value).setConf(conf);
              }
              value.readFields(in);                 // read value
            } finally {
              readingCall = null;
            }
            call.setResult(value, null);
          }

View Full Code Here


        Call call = calls.get(id);


        int state = in.readInt();     // read call status
        if (state == Status.SUCCESS.state) {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in);                 // read value
          call.setValue(value);
          calls.remove(id);
        } else if (state == Status.ERROR.state) {
          call.setException(new RemoteException(WritableUtils.readString(in),
                                                WritableUtils.readString(in)));

View Full Code Here

    
    // Read the data back and check
    LOG.info("Reading from the SequenceFile...");
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
    
    Writable key = (Writable)reader.getKeyClass().newInstance();
    Writable value = (Writable)reader.getValueClass().newInstance();
    
    int lc = 0;
    try {
      while (reader.next(key, value)) {
        assertEquals("key" + lc, key.toString());
        assertEquals("value" + lc, value.toString());
        lc ++;
      }
    } finally {
      reader.close();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Writable

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.merge.TestMergeJob

com.asakusafw.testtools.TestDataHolder

com.esri.hadoop.hive.serde.JsonSerde

com.facebook.giraph.hive.impl.output.HiveApiRecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.RecordWriterImpl

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.sap.hadoop.ds.list.ByteBasedListTest

com.sap.hadoop.ds.list.PartitionedByteBasedListTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.