Examples of org.apache.hadoop.io.ArrayWritable

org.apache.hadoop.io.ArrayWritable
A Writable for arrays containing instances of a class.

    int port = BSPNetUtils.getFreePort(12502);
    LOG.info("Got port = " + port);


    boolean result = syncClient
        .getInformation(syncClient.constructKey(job.getJobID(), "checkpoint",
            "" + bspTask.getPeerIndex()), new ArrayWritable(LongWritable.class));


    assertFalse(result);


    bspTask.sync();
    // Superstep 1

View Full Code Here

    String writeKey = "job_checkpttest_0001/checkpoint/1/";


    Writable[] writableArr = new Writable[2];
    writableArr[0] = new LongWritable(3L);
    writableArr[1] = new LongWritable(5L);
    ArrayWritable arrWritable = new ArrayWritable(LongWritable.class);
    arrWritable.set(writableArr);
    syncClient.storeInformation(writeKey, arrWritable, true, null);


    String writePath = "checkpoint/job_checkpttest_0001/3/1";
    FSDataOutputStream out = dfs.create(new Path(writePath));
    for (int i = 0; i < 5; ++i) {

View Full Code Here

    }


    @Override
    public boolean storeInformation(String key, Writable value,
        boolean permanent, SyncEventListener listener) {
      ArrayWritable writables = (ArrayWritable) value;
      long step = ((LongWritable) writables.get()[0]).get();
      long count = ((LongWritable) writables.get()[1]).get();


      LOG.info("SyncClient Storing value step = " + step + " count = " + count
          + " for key " + key);
      valueMap.put(key, value);
      return true;

View Full Code Here

        LOG.debug(list);
      }


      if (taskProgress.length == this.tasks.length) {
        for (int i = 0; i < taskProgress.length; ++i) {
          ArrayWritable progressInformation = new ArrayWritable(
              LongWritable.class);
          boolean result = this.masterSyncClient.getInformation(
              this.masterSyncClient.constructKey(jobId, "checkpoint",
                  taskProgress[i]), progressInformation);


          if (!result) {
            lowestSuperstepNumber = -1L;
            break;
          }


          Writable[] progressArr = progressInformation.get();
          LongWritable superstepProgress = (LongWritable) progressArr[0];


          if (superstepProgress != null) {
            if (superstepProgress.get() < lowestSuperstepNumber) {
              lowestSuperstepNumber = superstepProgress.get();

View Full Code Here


    @Override
    public TaskStatus.State onPeerInitialized(TaskStatus.State state)
        throws Exception {
      if (this.superstep >= 0 && state.equals(TaskStatus.State.RECOVERING)) {
        ArrayWritable progressArr = new ArrayWritable(LongWritable.class);
        boolean result = this.syncClient.getInformation(
            this.syncClient.constructKey(job.getJobID(), "checkpoint",
                String.valueOf(peer.getPeerIndex())), progressArr);


        if (!result) {
          throw new IOException("No data found to restore peer state.");
        }


        Writable[] progressInfo = progressArr.get();
        long superstepProgress = ((LongWritable) progressInfo[0]).get();
        long numMessages = ((LongWritable) progressInfo[1]).get();


        if (LOG.isDebugEnabled()) {
          LOG.debug("Got sstep =" + superstepProgress + " numMessages = "

View Full Code Here

            this.checkpointStream = null;
          }


          lastCheckPointStep = peer.getSuperstepCount();


          ArrayWritable writableArray = new ArrayWritable(LongWritable.class);
          Writable[] writeArr = new Writable[2];
          writeArr[0] = new LongWritable(lastCheckPointStep);
          writeArr[1] = new LongWritable(checkpointMessageCount);
          writableArray.set(writeArr);


          if (LOG.isDebugEnabled()) {
            LOG.debug("Storing lastCheckPointStep = " + lastCheckPointStep
                + " checkpointMessageCount = " + checkpointMessageCount
                + " for peer = " + String.valueOf(peer.getPeerIndex()));

View Full Code Here

      assertEquals(false, result);
      
      Writable[] writableArr = new Writable[2];
      writableArr[0] = new LongWritable(3L);
      writableArr[1] = new LongWritable(5L);
      ArrayWritable arrWritable = new ArrayWritable(LongWritable.class);
      arrWritable.set(writableArr);
      masterClient.storeInformation(
          masterClient.constructKey(jobID, "info", "level3"), 
          arrWritable, true, null);
      
      ArrayWritable valueHolder = new ArrayWritable(LongWritable.class);
      
      boolean getResult = masterClient.getInformation(
          masterClient.constructKey(jobID, "info", "level3"), valueHolder);
      
      assertTrue(getResult);
      
      assertEquals(arrWritable.get()[0], valueHolder.get()[0]);
      assertEquals(arrWritable.get()[1], valueHolder.get()[1]);
      
      Log.info("Passed array writable test");
      done = true;


    } catch (Exception e) {

View Full Code Here

      FSImageSerialization.writeString(path, out);
      FSImageSerialization.writeShort(replication, out);
      FSImageSerialization.writeLong(mtime, out);
      FSImageSerialization.writeLong(atime, out);
      FSImageSerialization.writeLong(blockSize, out);
      new ArrayWritable(Block.class, blocks).write(out);
      permissions.write(out);


      if (this.opCode == OP_ADD) {
        FSImageSerialization.writeString(clientName,out);
        FSImageSerialization.writeString(clientMachine,out);

View Full Code Here

      DeprecatedUTF8 info[] = new DeprecatedUTF8[srcs.length];
      int idx = 0;
      for(int i=0; i<srcs.length; i++) {
        info[idx++] = new DeprecatedUTF8(srcs[i]);
      }
      new ArrayWritable(DeprecatedUTF8.class, info).write(out);


      FSImageSerialization.writeLong(timestamp, out);
      
      // rpc ids
      writeRpcIds(rpcClientId, rpcCallId, out);

View Full Code Here

          byte opcode = in.readByte();
          numEdits++;
          switch (opcode) {
          case OP_ADD: {
            UTF8 name = new UTF8();
            ArrayWritable aw = null;
            Writable writables[];
            // version 0 does not support per file replication
            if( logVersion >= 0 )
              name.readFields(in);  // read name only
            else {  // other versions do
              // get name and replication
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get(); 
              if( writables.length != 2 )
                throw new IOException("Incorrect data fortmat. " 
                    + "Name & replication pair expected");
              name = (UTF8) writables[0];
              replication = Short.parseShort(
                  ((UTF8)writables[1]).toString());
              replication = adjustReplication( replication, conf );
            }
            // get blocks
            aw = new ArrayWritable(Block.class);
            aw.readFields(in);
            writables = aw.get();
            Block blocks[] = new Block[writables.length];
            System.arraycopy(writables, 0, blocks, 0, blocks.length);
            // add to the file tree
            fsDir.unprotectedAddFile(name, blocks, replication );
            break;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.ArrayWritable

co.nubetech.hiho.dedup.TestHashUtility

com.mozilla.hadoop.hbase.mapreduce.MultiScanTableMapReduceUtil

edu.berkeley.chukwa_xtrace.XtrExtract$Reduce

net.sf.katta.lib.mapfile.TextArrayWritable

org.apache.hadoop.dfs.FSEditLog

org.apache.hadoop.hdfs.server.namenode.FSEditLog

org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$AddCloseOp

org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$ConcatDeleteOp

org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$DeleteOp

org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$MkdirOp

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.