Examples of com.pinterest.secor.io.KeyValue

com.pinterest.secor.io.KeyValue
Generic Object used to read next message from various file reader implementations @author Praveen Murugesan (praveen@uber.com)

    }


    @Override
    public KeyValue next() throws IOException {
        if (mReader.next(mKey, mValue)) {
            return new KeyValue(mKey.get(), mValue.getBytes());
        } else {
            return null;
        }
    }

View Full Code Here

                            "Non-empty message without delimiter");
                }
            }
            messageBuffer.write(nextByte);
        }
        return new KeyValue(this.mOffset++, messageBuffer.toByteArray());
    }

View Full Code Here

            public KeyValue answer(InvocationOnMock invocation)
                    throws Throwable {
                if (mCallCount == 2) {
                    return null;
                }
                return new KeyValue(20 + mCallCount++, null);
            }
        });


        PowerMockito.mockStatic(IdUtil.class);
        Mockito.when(IdUtil.getLocalMessageDir())

View Full Code Here

            if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) {
                codec = CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec());
                extension = codec.getDefaultExtension();
            }
            reader = createReader(srcPath, codec);
            KeyValue keyVal;
            while ((keyVal = reader.next()) != null) {
                if (keyVal.getKey() >= startOffset) {
                    if (writer == null) {
                        String localPrefix = mConfig.getLocalPath() + '/' +
                            IdUtil.getLocalMessageDir();
                        dstPath = new LogFilePath(localPrefix, srcPath.getTopic(),
                                                  srcPath.getPartitions(), srcPath.getGeneration(),

View Full Code Here

                                                           message.getKafkaPartition());
        long offset = mOffsetTracker.getAdjustedCommittedOffsetCount(topicPartition);
        LogFilePath path = new LogFilePath(mLocalPrefix, mConfig.getGeneration(), offset, message,
            mFileExtension);
        FileReaderWriter writer = mFileRegistry.getOrCreateWriter(path, mCodec);
        writer.write(new KeyValue(message.getOffset(), message.getPayload()));
        LOG.debug("appended message " + message + " to file " + path.getLogFilePath() +
                  ".  File length " + writer.getLength());
    }

View Full Code Here

TOP

Related Classes of com.pinterest.secor.io.KeyValue

com.pinterest.secor.io.impl.DelimitedTextFileReaderWriter

com.pinterest.secor.io.impl.SequenceFileReaderWriter

com.pinterest.secor.uploader.Uploader

com.pinterest.secor.uploader.UploaderTest

com.pinterest.secor.writer.MessageWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.