Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.Writable


        File result = folder.newFile();
        Path p = new Path(new File(path).toURI());
        FileSystem fs = p.getFileSystem(new Configuration());
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, fs.getConf());
        try {
            Writable buffer = (Writable) reader.getValueClass().newInstance();
            ModelOutput<Writable> output = (ModelOutput<Writable>) TemporaryStorage.openOutput(
                    fs.getConf(),
                    reader.getValueClass(),
                    new BufferedOutputStream(new FileOutputStream(result)));
            try {
View Full Code Here


        }

        // 各レコードを比較
        Iterator<Writable> expectIterator = expect.iterator();
        Iterator<Writable> actualIterator = actual.iterator();
        Writable expectRow = null;
        Writable actualRow = null;
        for (;;) {
            // expect, actualのどちらかしか存在しない場合
            if (!expectIterator.hasNext() && expectRow == null) {
                if (actualRow != null) {
                    fail(Type.NO_EXPECT_RECORD, null, actualRow);
View Full Code Here

     * @param expected
     * @throws Exception
     */
    private void testField(List<Writable> list, String getterName,  DATA data, Object expected) throws Exception {
        int index = data.getRownum() - 1; // Excelシートの一行目がリストの最初に入るため -1 する必要がある
        Writable modelObject = list.get(index);
        Method method = modelObject.getClass().getMethod(getterName);
        Object actual = method.invoke(modelObject);
        String format = "テストデータ: %s, getter名 %s";
        String message = String.format(format, data.getComment(), getterName);
        assertEquals("型の一致の確認 " + message, expected.getClass(), actual.getClass());
        assertEquals("値の一致の確認 " + message, expected, actual);
View Full Code Here

            rownum++;
            HSSFRow row = sheet.getRow(rownum);
            if (isEmpty(row)) {
                break;
            }
            Writable model;
            try {
                model = (Writable) modelClass.newInstance();
            } catch (InstantiationException e) {
                throw new RuntimeException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
            for (int col = 0; col < columnInfos.size(); col++) {
                HSSFCell cell = row.getCell(col, Row.CREATE_NULL_AS_BLANK);
                MySqlDataType type = columnInfos.get(col).getDataType();
                ValueOption<?> vo;
                switch (type) {
                case CHAR:
                case VARCHAR:
                    vo = getStringOption(cell);
                    break;
                case DATE:
                    vo = getDateOption(cell);
                    break;
                case DATETIME:
                case TIMESTAMP:
                    vo = getDateTimeOption(cell);
                    break;
                case DECIMAL:
                    vo = getDecimalOption(cell);
                    break;
                case TINY_INT:
                    vo = getByteOption(cell);
                    break;
                case SMALL_INT:
                    vo = getShortOption(cell);
                    break;
                case INT:
                    vo = getIntOption(cell);
                    break;
                case LONG:
                    vo = getLongOption(cell);
                    break;
                default:
                    throw new RuntimeException("Unsupported data type: " + type);
                }
                try {
                    String setterName = columnInfos.get(col).getSetterName();
                    Method setter = model.getClass().getMethod(setterName, vo.getClass());
                    setter.invoke(model, vo);
                } catch (IllegalAccessException e) {
                    throw new RuntimeException(e);
                } catch (InvocationTargetException e) {
                    throw new RuntimeException(e);
View Full Code Here

     * @return 復元したデータ
     */
    @SuppressWarnings("unchecked")
    protected <T> T restore(T value) {
        assertThat(value, instanceOf(Writable.class));
        Writable writable = (Writable) value;
        try {
            ByteArrayOutputStream write = new ByteArrayOutputStream();
            ObjectOutputStream out = new ObjectOutputStream(write);
            writable.write(out);
            out.close();

            ByteArrayInputStream read = new ByteArrayInputStream(write.toByteArray());
            ObjectInputStream in = new ObjectInputStream(read);
            Writable copy = writable.getClass().newInstance();
            copy.readFields(in);
            assertThat(in.read(), is(-1));
            assertThat(copy, is((Writable) value));
            assertThat(copy.hashCode(), is(value.hashCode()));
            return (T) copy;
        } catch (Exception e) {
            throw new AssertionError(e);
        }
    }
View Full Code Here

            //noinspection ThrowableInstanceNeverThrown
            call.setException(new RemoteException(WritableUtils.readString(in),
                WritableUtils.readString(in)));
          }
        } else {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in);                 // read value
          // it's possible that this call may have been cleaned up due to a RPC
          // timeout, so check if it still exists before setting the value.
          if (call != null) {
            call.setValue(value);
          }
View Full Code Here

      if (this.isError)
        return;
      if (errorClass != null) {
        this.isError = true;
      }
      Writable result = null;
      if (value instanceof Writable) {
        result = (Writable) value;
      } else {
        /* We might have a null value and errors. Avoid creating a
         * HbaseObjectWritable, because the constructor fails on null. */
        if (value != null) {
          result = new HbaseObjectWritable(value);
        }
      }

      int size = BUFFER_INITIAL_SIZE;
      if (result instanceof WritableWithSize) {
        // get the size hint.
        WritableWithSize ohint = (WritableWithSize) result;
        long hint = ohint.getWritableSize() + Bytes.SIZEOF_BYTE +
          (2 * Bytes.SIZEOF_INT);
        if (hint > Integer.MAX_VALUE) {
          // oops, new problem.
          IOException ioe =
            new IOException("Result buffer size too large: " + hint);
          errorClass = ioe.getClass().getName();
          error = StringUtils.stringifyException(ioe);
        } else {
          size = (int)hint;
        }
      }

      ByteBufferOutputStream buf = new ByteBufferOutputStream(size);
      DataOutputStream out = new DataOutputStream(buf);
      try {
        // Call id.
        out.writeInt(this.id);
        // Write flag.
        byte flag = (error != null)?
          ResponseFlag.getErrorAndLengthSet(): ResponseFlag.getLengthSetOnly();
        out.writeByte(flag);
        // Place holder for length set later below after we
        // fill the buffer with data.
        out.writeInt(0xdeadbeef);
        out.writeInt(status.state);
      } catch (IOException e) {
        errorClass = e.getClass().getName();
        error = StringUtils.stringifyException(e);
      }

      try {
        if (error == null) {
          result.write(out);
        } else {
          WritableUtils.writeString(out, errorClass);
          WritableUtils.writeString(out, error);
        }
      } catch (IOException e) {
View Full Code Here

            "Call queue is full, is ipc.server.max.callqueue.size too small?");
        responder.doRespond(callTooBig);
        return;
      }

      Writable param;
      try {
        param = ReflectionUtils.newInstance(paramClass, conf);//read param
        param.readFields(dis);
      } catch (Throwable t) {
        LOG.warn("Unable to read call parameters for client " +
                 getHostAddress(), t);
        final Call readParamsFailedCall =
          new Call(id, null, this, responder, callSize);
View Full Code Here

            LOG.debug(getName() + ": has #" + call.id + " from " +
                      call.connection);

          String errorClass = null;
          String error = null;
          Writable value = null;

          CurCall.set(call);
          try {
            if (!started)
              throw new ServerNotRunningYetException("Server is not running yet");
View Full Code Here

        final String err = "Unknown rpc kind in rpc header"  +
            header.getRpcKind();
        throw new WrappedRpcServerException(
            RpcErrorCodeProto.FATAL_INVALID_RPC_HEADER, err);  
      }
      Writable rpcRequest;
      try { //Read the rpc request
        rpcRequest = ReflectionUtils.newInstance(rpcRequestClass, conf);
        rpcRequest.readFields(dis);
      } catch (Throwable t) { // includes runtime exception from newInstance
        LOG.warn("Unable to read call parameters for client " +
                 getHostAddress() + "on connection protocol " +
            this.protocolName + " for rpcKind " + header.getRpcKind(),  t);
        String err = "IPC server unable to read call parameters: "+ t.getMessage();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.Writable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.