Examples of com.linkedin.camus.coders.CamusWrapper

com.linkedin.camus.coders.CamusWrapper
Container for messages. Enables the use of a custom message decoder with knowledge of where these values are stored in the message schema @author kgoodhop @param < R> The type of decoded payload

            reader.close();
        }
    }


    private CamusWrapper getWrappedRecord(String topicName, byte[] payload) throws IOException {
        CamusWrapper r = null;
        try {
            r = decoder.decode(payload);
        } catch (Exception e) {
            if (!skipSchemaErrors) {
                throw new IOException(e);

View Full Code Here

                          + ". Expected " + key.getChecksum(),  
                          key.getOffset());
                    }


                    long tempTime = System.currentTimeMillis();
                    CamusWrapper wrapper;
                    try {
                        wrapper = getWrappedRecord(key.getTopic(), bytes);
                    } catch (Exception e) {
                        if (exceptionCount < getMaximumDecoderExceptionsToPrint(context)) {
                            mapperContext.write(key, new ExceptionWritable(e));
                            exceptionCount++;
                        } else if (exceptionCount == getMaximumDecoderExceptionsToPrint(context)) {
                            exceptionCount = Integer.MAX_VALUE; //Any random value
                            log.info("The same exception has occured for more than " + getMaximumDecoderExceptionsToPrint(context) + " records. All further exceptions will not be printed");
                        }
                        continue;
                    }


                    if (wrapper == null) {
                        mapperContext.write(key, new ExceptionWritable(new RuntimeException(
                                "null record")));
                        continue;
                    }


                    long timeStamp = wrapper.getTimestamp();
                    try {
                        key.setTime(timeStamp);
                        key.addAllPartitionMap(wrapper.getPartitionMap());
                        setServerService();
                    } catch (Exception e) {
                        mapperContext.write(key, new ExceptionWritable(e));
                        continue;
                    }

View Full Code Here

          dataWriters.clear();
          currentTopic = key.getTopic();
        }


        committer.addCounts(key);
        CamusWrapper value = (CamusWrapper) val;
        String workingFileName = EtlMultiOutputFormat.getWorkingFileName(context, key);
        if (!dataWriters.containsKey(workingFileName))
        {
          dataWriters.put(workingFileName, getDataRecordWriter(context, workingFileName, value));
          log.info("Writing to data file: " + workingFileName);

View Full Code Here

        JsonStringMessageDecoder testDecoder = new JsonStringMessageDecoder();
        testDecoder.init(testProperties, "testTopic");
        String payload = "{\"timestamp\":  " + expectedTimestamp + ", \"myData\": \"myValue\"}";
        byte[] bytePayload = payload.getBytes();


        CamusWrapper actualResult = testDecoder.decode(bytePayload);
        long actualTimestamp = actualResult.getTimestamp();
        assertEquals(expectedTimestamp, actualTimestamp);
    }

View Full Code Here


        JsonStringMessageDecoder testDecoder = new JsonStringMessageDecoder();
        testDecoder.init(testProperties, "testTopic");
        String payload = "{\"timestamp\":  " + testTimestamp + ", \"myData\": \"myValue\"}";
        byte[] bytePayload = payload.getBytes();
        CamusWrapper actualResult = testDecoder.decode(bytePayload);
        long actualTimestamp = actualResult.getTimestamp();


        assertEquals(expectedTimestamp, actualTimestamp);
    }

View Full Code Here


        JsonStringMessageDecoder testDecoder = new JsonStringMessageDecoder();
        testDecoder.init(testProperties, "testTopic");
        String payload = "{\"timestamp\":  \"" + testTimestamp + "\", \"myData\": \"myValue\"}";
        byte[] bytePayload = payload.getBytes();
        CamusWrapper actualResult = testDecoder.decode(bytePayload);
        long actualTimestamp = actualResult.getTimestamp();


        assertEquals(expectedTimestamp, actualTimestamp);


    }

View Full Code Here

TOP

Related Classes of com.linkedin.camus.coders.CamusWrapper

com.linkedin.camus.etl.kafka.coders.TestJsonStringMessageDecoder

com.linkedin.camus.etl.kafka.mapred.EtlMultiOutputRecordWriter

com.linkedin.camus.etl.kafka.mapred.EtlRecordReader

org.apache.hadoop.io.MapWritable

org.apache.hadoop.io.Text

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.