Package org.apache.avro.generic

Examples of org.apache.avro.generic.GenericRecord


    runImport(getArgv(
        "--inline-lob-limit", "1", "--compression-codec", CodecMap.DEFLATE));

    Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
    DataFileReader<GenericRecord> reader = read(outputFile);
    GenericRecord record = reader.next();

    // Verify that the data block of the Avro file is compressed with deflate
    // codec.
    assertEquals(CodecMap.DEFLATE,
        reader.getMetaString(DataFileConstants.CODEC));

    // Verify that the reference file is written in Avro bytes.
    ByteBuffer buf = (ByteBuffer) record.get(getColName(0));
    String returnVal = new String(buf.array());
    String expectedStart = "externalLob(lf,_lob/large_obj";
    String expectedEnd = getTableNum() + "_m_0000000.lob,68,"
      + data.length() + ")";
View Full Code Here


    runImport(getArgv());

    Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
    DataFileReader<GenericRecord> reader = read(outputFile);
    GenericRecord record = reader.next();

    // Verify that all columns are imported correctly.
    ByteBuffer buf = (ByteBuffer) record.get(getColName(0));
    String returnVal = new String(buf.array());

    assertEquals(getColName(0), expectedVal1, returnVal);

    buf = (ByteBuffer) record.get(getColName(1));
    returnVal = new String(buf.array());

    assertEquals(getColName(1), expectedVal2, returnVal);

    buf = (ByteBuffer) record.get(getColName(2));
    returnVal = new String(buf.array());

    assertEquals(getColName(2), expectedVal3, returnVal);
  }
View Full Code Here

      return keySrc.compareTo(keyDst);
    }

    private long getDestTxnId(ResultSet bsRes) throws SQLException
    {
      GenericRecord avroRec = _auditor.getGenericRecord(bsRes, _decoder);
      if (avroRec == null)
      {
        LOG.error("No avro record skipping");
        return 0;
      }
      Object txnId = avroRec.get("txn");
      if (txnId == null)
      {
        LOG.error("Could not find a field called 'txn' in avro event in bootstrap db");
        return 0;
      }
View Full Code Here

   *        and will allow us to use something like org.apache.avro.ipc.ByteBufferInputStream
   *        to avoid the data copy to a temp array.  (https://rb.corp.linkedin.com/r/172879/)
   */
  public GenericRecord getGenericRecord(byte[] valueBytes, Schema schema, GenericRecord reuse)
  {
    GenericRecord result = null;
    try
    {
      binDecoder.set(DecoderFactory.defaultFactory().createBinaryDecoder(valueBytes, binDecoder.get()));
      GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
      result = reader.read(reuse, binDecoder.get());
View Full Code Here

    return null;
  }

  public void dumpMetadata(DbusEvent e, FileChannel writeChannel)
  {
    GenericRecord genericRecord = this.getMetadata(e, null);
    if( genericRecord == null ) //no metadata
      return;

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try
    {
      String metadataInfo = genericRecord.toString() + "\n";
      baos.write(metadataInfo.getBytes("UTF-8"));
      ByteBuffer writeBuffer = ByteBuffer.wrap(baos.toByteArray());
      writeChannel.write(writeBuffer);
    }
    catch (UnsupportedEncodingException e1)
View Full Code Here

    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();

    for (int i = 0; i < dbUpdates.size(); ++i)
    {
      GenericRecord record = null;
      TransactionState.PerSourceTransactionalUpdate perSourceUpdate = dbUpdates.get(i);
      short sourceId = (short)perSourceUpdate.getSourceId();
      // prepare stats collection per source
      EventSourceStatistics perSourceStats = getSource(sourceId).getStatisticsBean();

      Iterator<DbUpdateState.DBUpdateImage> dbUpdateIterator = perSourceUpdate.getDbUpdatesSet().iterator();
      int eventsInDbUpdate = 0;
      long dbUpdatesEventsSize = 0;
      long startDbUpdatesMs = System.currentTimeMillis();

      while(dbUpdateIterator.hasNext())  //TODO verify if there is any case where we need to rollback.
      {
        DbUpdateState.DBUpdateImage dbUpdate =  dbUpdateIterator.next();

        //Construct the Databus Event key, determine the key type and construct the key
        Object keyObj = obtainKey(dbUpdate);
        DbusEventKey eventKey = new DbusEventKey(keyObj);

        //Get the logicalparition id
        PartitionFunction partitionFunction = _partitionFunctionHashMap.get((int)sourceId);
        short lPartitionId = partitionFunction.getPartition(eventKey);

        record = dbUpdate.getGenericRecord();
        //Write the event to the buffer
        if (record == null)
          throw new DatabusException("Cannot write event to buffer because record = " + record);

        if(record.getSchema() == null)
          throw new DatabusException("The record does not have a schema (null schema)");

        try
        {
          //Collect stats on number of dbUpdates for one source
          eventsInDbUpdate++;

          //Count of all the events in the current transaction
          eventsInTransactionCount++;
          // Serialize the row
          ByteArrayOutputStream bos = new ByteArrayOutputStream();
          Encoder encoder = new BinaryEncoder(bos);
          GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(
              record.getSchema());
          writer.write(record, encoder);
          byte[] serializedValue = bos.toByteArray();

          //Get the md5 for the schema
          SchemaId schemaId = SchemaId.createWithMd5(dbUpdate.getSchema());
View Full Code Here

    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(
        1);
    keyPairs.add(kp);

    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");

    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,
        DbUpdateState.DBUpdateImage.OpType.INSERT,false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(
View Full Code Here

      {
        ColumnsState.KeyPair kp = new ColumnsState.KeyPair(k, keyType);
        ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(
            1);
        keyPairs.add(kp);
        GenericRecord gr = new GenericData.Record(s);
        gr.put(keyName, k);
        gr.put(valName, keyVals.get(k));

        DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,
                                              DbUpdateState.DBUpdateImage.OpType.INSERT,false);
        db.add(dbi);
        TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(
View Full Code Here

      {
        ColumnsState.KeyPair kp = new ColumnsState.KeyPair(k, keyType);
        ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(
            1);
        keyPairs.add(kp);
        GenericRecord gr = new GenericData.Record(s);
        gr.put(keyName, k);

        DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,
                                              DbUpdateState.DBUpdateImage.OpType.INSERT,false);
        db.add(dbi);
        TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(
View Full Code Here

        numEvents);
    keyPairs.add(kp1);

    Schema s = Schema.parse(avroSchema2);

    GenericRecord gr1 = new GenericData.Record(s);
    gr1.put("name1", "phani1");
    gr1.put("name2", "boris1");

    GenericRecord gr2 = new GenericData.Record(s);
    gr2.put("name1", "phani2");
    gr2.put("name2", "boris2");

    GenericRecord gr3 = new GenericData.Record(s);
    gr3.put("name1", "phani3");
    gr3.put("name2", "boris3");

    GenericRecord gr4 = new GenericData.Record(s);
    gr4.put("name1", "phani4");
    gr4.put("name2", "boris4");

    GenericRecord gr5 = new GenericData.Record(s);
    gr5.put("name1", "phani5");
    gr5.put("name2", "boris5");


    DBUpdateImage dbi1 = new DBUpdateImage(keyPairs, scn, gr1, s,
        DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi2 = new DBUpdateImage(keyPairs, scn, gr2, s,
View Full Code Here

TOP

Related Classes of org.apache.avro.generic.GenericRecord

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.