Examples of org.apache.hadoop.io.Writable

org.apache.hadoop.io.Writable
A simple, efficient, serialization protocol, based on {@link DataInput} and{@link DataOutput}.
Implementations typically implement a static read(DataInput) method which constructs a new instance, calls {@link #readFields(DataInput)}, and returns the instance.


    WebPage page = null;
    inlinkedScoreData.clear();
    
    for (NutchWritable nutchWritable : values) {
      Writable val = nutchWritable.get();
      if (val instanceof WebPageWritable) {
        page = ((WebPageWritable) val).getWebPage();
      } else {
        inlinkedScoreData.add((ScoreDatum) val);
        if (inlinkedScoreData.size() >= maxLinks) {

View Full Code Here


        Call call = calls.get(id);


        int state = in.readInt();     // read call status
        if (state == Status.SUCCESS.state) {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in);                 // read value
          call.setValue(value);
          calls.remove(id);
        } else if (state == Status.ERROR.state) {
          call.setException(new RemoteException(WritableUtils.readString(in),
                                                WritableUtils.readString(in)));

View Full Code Here

            //noinspection ThrowableInstanceNeverThrown
            call.setException(new RemoteException(WritableUtils.readString(in),
                WritableUtils.readString(in)));
          }
        } else {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in);                 // read value
          // it's possible that this call may have been cleaned up due to a RPC
          // timeout, so check if it still exists before setting the value.
          if (call != null) {
            call.setValue(value);
          }

View Full Code Here

      int id = dis.readInt();                    // try to read an id
        
      if (LOG.isDebugEnabled())
        LOG.debug(" got #" + id);
            
      Writable param = (Writable)ReflectionUtils.newInstance(paramClass, conf);           // read param
      param.readFields(dis);        
        
      Call call = new Call(id, param, this);
      synchronized (callQueue) {
        if (callQueue.size() >= maxQueueSize) {
          Call oldCall = callQueue.removeFirst();

View Full Code Here

            LOG.debug(getName() + ": has #" + call.id + " from " +
                      call.connection);
          
          String errorClass = null;
          String error = null;
          Writable value = null;
          
          CurCall.set(call);
          UserGroupInformation previous = UserGroupInformation.getCurrentUGI();
          UserGroupInformation.setCurrentUGI(call.connection.ticket);
          try {
            value = call(call.param, call.receivedTime);             // make the call
          } catch (Throwable e) {
            LOG.info(getName()+", call "+call+": error: " + e, e);
            errorClass = e.getClass().getName();
            error = StringUtils.stringifyException(e);
          }
          UserGroupInformation.setCurrentUGI(previous);
          CurCall.set(null);


          buf.reset();
          DataOutputStream out = new DataOutputStream(buf);
          out.writeInt(call.id);                // write call id
          out.writeBoolean(error != null);      // write error flag


          if (error == null) {
            value.write(out);
          } else {
            WritableUtils.writeString(out, errorClass);
            WritableUtils.writeString(out, error);
          }
          call.setResponse(ByteBuffer.wrap(buf.toByteArray()));

View Full Code Here

        reporter.progress();
      }
    }
    
    private void spill(RawKeyValueIterator resultIter) throws IOException {
      Writable key = null;
      Writable value = null;


      try {
        // indicate progress, since constructor may take a while (because of 
        // user code) 
        reporter.progress();
        key = (WritableComparable)ReflectionUtils.newInstance(keyClass, job);
        value = (Writable)ReflectionUtils.newInstance(valClass, job);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }


      DataInputBuffer keyIn = new DataInputBuffer();
      DataInputBuffer valIn = new DataInputBuffer();
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(), 
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);
        writer.append(key, value);
        reporter.progress();
      }
    }

View Full Code Here

        int entries = in.readInt();


        // Then read each key/value pair


        for (int i = 0; i < entries; i++) {
            Writable key = (Writable) ReflectionUtils.newInstance(getClass(in.readByte()), getConf());


            key.readFields(in);


            Writable value = (Writable) ReflectionUtils.newInstance(getClass(in.readByte()), getConf());


            value.readFields(in);
            instance.put(key, value);
        }
    }

View Full Code Here


        try {
            Iterator<Entry<Writable, Writable>> i = entrySet().iterator();
            while (i.hasNext()) {
                Entry<Writable, Writable> e = i.next();
                Writable key = e.getKey();
                Writable value = e.getValue();
                if (value == null) {
                    if (!(m.get(key) == null && m.containsKey(key)))
                        return false;
                }
                else {
                    if (!value.equals(m.get(key)))
                        return false;
                }
            }
        } catch (ClassCastException unused) {
            return false;

View Full Code Here


        StringBuilder sb = new StringBuilder();
        sb.append('{');
        for (;;) {
            Entry<Writable, Writable> e = i.next();
            Writable key = e.getKey();
            Writable value = e.getValue();
            sb.append(key == this ? "(this Map)" : key);
            sb.append('=');
            if (value instanceof ArrayWritable) {
                sb.append(Arrays.toString(((ArrayWritable) value).get()));
            }

View Full Code Here

    
    FileSystem outputFS = getFileSystem();
    Path outputPath = new Path(outputFS.getHomeDirectory(), "output/part-r-00000");
    Configuration conf = new Configuration();
    SequenceFile.Reader reader = new SequenceFile.Reader(outputFS, outputPath, conf);
    Writable writableKey = (Writable)
    ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    Writable writableValue = (Writable)
    ReflectionUtils.newInstance(reader.getValueClass(), conf);
    List<Text> expectedOutput = new ArrayList<Text>();
    expectedOutput.add(new Text("Xavier Wilson,Mason Holloway,Carlos Johnston,Martin Noel,Drake Mckinney"));
    expectedOutput.add(new Text("Kennedy Bailey,Jerome Perry,David Cabrera,Edan Fleming,Orlando Tyson"));
    expectedOutput.add(new Text("Drake Mckinney,Murphy Baird,Theodore Lindsey,Nehru Wilcox,Harper Klein"));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Writable

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.merge.TestMergeJob

com.asakusafw.testtools.TestDataHolder

com.esri.hadoop.hive.serde.JsonSerde

com.facebook.giraph.hive.impl.output.HiveApiRecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.RecordWriterImpl

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.sap.hadoop.ds.list.ByteBasedListTest

com.sap.hadoop.ds.list.PartitionedByteBasedListTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.