Examples of FileReaderWriter

com.pinterest.secor.io.FileReaderWriter
Generic file reader/writer interface for all secor files All implementations should define the constructor with the signature: FileReaderWriter(LogFilePath path, CompressionCodec codec, FileReaderWriter.Type type) example: public SequenceFileReaderWriter(LogFilePath path, CompressionCodec codec, FileReaderWriter.Type type) @author Praveen Murugesan (praveen@uber.com)

Examples of com.pinterest.secor.io.FileReaderWriter

        HashSet<LogFilePath> logFilePaths = new HashSet<LogFilePath>();
        logFilePaths.add(mLogFilePath);
        Mockito.when(mFileRegistry.getPaths(mTopicPartition)).thenReturn(
                logFilePaths);


        FileReaderWriter reader = mUploader.getReader();


        Mockito.when(reader.next()).thenAnswer(new Answer<KeyValue>() {
            private int mCallCount = 0;


            @Override
            public KeyValue answer(InvocationOnMock invocation)
                    throws Throwable {
                if (mCallCount == 2) {
                    return null;
                }
                return new KeyValue(20 + mCallCount++, null);
            }
        });


        PowerMockito.mockStatic(IdUtil.class);
        Mockito.when(IdUtil.getLocalMessageDir())
                .thenReturn("some_message_dir");


        FileReaderWriter writer = Mockito.mock(FileReaderWriter.class);
        LogFilePath dstLogFilePath = new LogFilePath(
                "/some_parent_dir/some_message_dir",
                "/some_parent_dir/some_message_dir/some_topic/some_partition/"
                        + "some_other_partition/10_0_00000000000000000021");
        Mockito.when(mFileRegistry.getOrCreateWriter(dstLogFilePath, null))

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter


    private void createWriter() throws Exception {
        PowerMockito.mockStatic(FileUtil.class);


        PowerMockito.mockStatic(ReflectionUtil.class);
        FileReaderWriter writer = Mockito.mock(FileReaderWriter.class);
        Mockito.when(
                ReflectionUtil.createFileReaderWriter(
                        Mockito.any(String.class),
                        Mockito.any(LogFilePath.class),
                        Mockito.any(CompressionCodec.class),
                        Mockito.any(FileReaderWriter.Type.class)))
                .thenReturn(writer);


        Mockito.when(writer.getLength()).thenReturn(123L);


        FileReaderWriter createdWriter = mRegistry.getOrCreateWriter(
                mLogFilePath, null);
        assertTrue(createdWriter == writer);
    }

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter


    private void createCompressedWriter() throws Exception {
        PowerMockito.mockStatic(FileUtil.class);


        PowerMockito.mockStatic(ReflectionUtil.class);
        FileReaderWriter writer = Mockito.mock(FileReaderWriter.class);
        Mockito.when(
                ReflectionUtil.createFileReaderWriter(
                        Mockito.any(String.class),
                        Mockito.any(LogFilePath.class),
                        Mockito.any(CompressionCodec.class),
                        Mockito.any(FileReaderWriter.Type.class)))
                .thenReturn(writer);


        Mockito.when(writer.getLength()).thenReturn(123L);


        FileReaderWriter createdWriter = mRegistry.getOrCreateWriter(
                mLogFilePathGz, new GzipCodec());
        assertTrue(createdWriter == writer);
    }

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter

    /**
     * Delete writer for a given topic partition.  Underlying file is not removed.
     * @param path The path to remove the writer for.
     */
    public void deleteWriter(LogFilePath path) throws IOException {
        FileReaderWriter writer = mWriters.get(path);
        if (writer == null) {
            LOG.warn("No writer found for path " + path.getLogFilePath());
        } else {
            LOG.info("Deleting writer for path " + path.getLogFilePath());
            writer.close();
            mWriters.remove(path);
            mCreationTimes.remove(path);
        }
    }

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter

     */
    public long getSize(TopicPartition topicPartition) throws IOException {
        Collection<LogFilePath> paths = getPaths(topicPartition);
        long result = 0;
        for (LogFilePath path : paths) {
            FileReaderWriter writer = mWriters.get(path);
            if (writer != null) {
                result += writer.getLength();
            }
        }
        StatsUtil.setLabel("secor.size." + topicPartition.getTopic() + "." +
                           topicPartition.getPartition(), Long.toString(result));
        return result;

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter


    private void trim(LogFilePath srcPath, long startOffset) throws Exception {
        if (startOffset == srcPath.getOffset()) {
            return;
        }
        FileReaderWriter reader = null;
        FileReaderWriter writer = null;
        LogFilePath dstPath = null;
        int copiedMessages = 0;
        // Deleting the writer closes its stream flushing all pending data to the disk.
        mFileRegistry.deleteWriter(srcPath);
        try {
            CompressionCodec codec = null;
            String extension = "";
            if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) {
                codec = CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec());
                extension = codec.getDefaultExtension();
            }
            reader = createReader(srcPath, codec);
            KeyValue keyVal;
            while ((keyVal = reader.next()) != null) {
                if (keyVal.getKey() >= startOffset) {
                    if (writer == null) {
                        String localPrefix = mConfig.getLocalPath() + '/' +
                            IdUtil.getLocalMessageDir();
                        dstPath = new LogFilePath(localPrefix, srcPath.getTopic(),
                                                  srcPath.getPartitions(), srcPath.getGeneration(),
                                                  srcPath.getKafkaPartition(), startOffset,
                                                  extension);
                        writer = mFileRegistry.getOrCreateWriter(dstPath,
                            codec);
                    }
                    writer.write(keyVal);
                    copiedMessages++;
                }
            }
        } finally {
            if (reader != null) {

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter

        TopicPartition topicPartition = new TopicPartition(message.getTopic(),
                                                           message.getKafkaPartition());
        long offset = mOffsetTracker.getAdjustedCommittedOffsetCount(topicPartition);
        LogFilePath path = new LogFilePath(mLocalPrefix, mConfig.getGeneration(), offset, message,
            mFileExtension);
        FileReaderWriter writer = mFileRegistry.getOrCreateWriter(path, mCodec);
        writer.write(new KeyValue(message.getOffset(), message.getPayload()));
        LOG.debug("appended message " + message + " to file " + path.getLogFilePath() +
                  ".  File length " + writer.getLength());
    }

View Full Code Here

Examples of com.pinterest.secor.io.FileReaderWriter

     * @return Writer for a given path.
     * @throws Exception 
     */
    public FileReaderWriter getOrCreateWriter(LogFilePath path, CompressionCodec codec)
            throws Exception {
        FileReaderWriter writer = mWriters.get(path);
        if (writer == null) {
            // Just in case.
            FileUtil.delete(path.getLogFilePath());
            FileUtil.delete(path.getLogFileCrcPath());
            TopicPartition topicPartition = new TopicPartition(path.getTopic(),

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.