Examples of org.apache.pig.impl.io.PigLineRecordReader

org.apache.pig.impl.io.PigLineRecordReader
This class is for Reading data line by Line in Pig It uses org.apache.hadoop.mapred.LineRecordReader for reading simple Text For BZip it uses a different class which does not do buffering.

    public Tuple getSampledTuple() throws IOException {     
        return getNext();
    }


    public void bindTo(String fileName, BufferedPositionedInputStream in, long offset, long end) throws IOException {
        this.in = new PigLineRecordReader( in, offset, end );
        this.end = end;
        
        // Since we are not block aligned we throw away the first
        // record and cound on a different instance to read it
        if (offset != 0) {

View Full Code Here

    final private static Charset utf8 = Charset.forName("UTF8");
    long end;
    private TupleFactory mTupleFactory = TupleFactory.getInstance();


    public void bindTo(String fileName, BufferedPositionedInputStream in, long offset, long end) throws IOException {
        this.in = new PigLineRecordReader( in, offset, end );
        this.end = end;
        // Since we are not block aligned we throw away the first
        // record and count on a different instance to read it
        if (offset != 0)
            getNext();

View Full Code Here

            
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( bzis );
            
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, 100 );


            Field lineReader = reader.getClass().getDeclaredField("lineReader");
            lineReader.setAccessible(true);
            Object lineReaderObj = lineReader.get(reader);            
            assertTrue( "Expected a PigLineRecordReader.BufferingLineReader", 
                    lineReaderObj instanceof PigLineRecordReader.LineReader );

View Full Code Here

            }
            ps.close();
            
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( is );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
            
            Text value = new Text();
            int counter = 0;
            while( reader.next(value) ) {
                assertTrue( "Invalid Text", value.toString().compareTo(text) == 0 );
                counter++;
            }
            assertEquals("Invalid number of lines", counter, LOOP_COUNT );
            testFile.deleteOnExit();

View Full Code Here

            ps.close();
            
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( bzis );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
            
            Text value = new Text();
            int counter = 0;
            while( reader.next(value) ) {
                assertTrue( "Invalid Text", value.toString().compareTo(text) == 0 );
                counter++;
            }
            assertEquals("Invalid number of lines", counter, LOOP_COUNT );
            testFile.deleteOnExit();

View Full Code Here

            }
            ps.close();
            
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( is );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
            
            Text value = new Text();
            int counter = 0;
            for( int i = 0; i < LOOP_COUNT / 2; i++ ) {
                reader.next(value);
                assertTrue( "Invalid Text", value.toString().compareTo(text) == 0 );
                counter++;
            }
            assertEquals( "Invalid bytes read", reader.getPosition() , counter * ( text.length() + 1 ) );        
            
            testFile.deleteOnExit();
            
        } catch (IOException e) {
            e.printStackTrace();

View Full Code Here

            CBZip2InputStream bzis = new CBZip2InputStream( is );
            
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( bzis );
            
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, 
                    ( text.length() + 1 ) * ( LOOP_COUNT/2 ) );
            
            Text value = new Text();
            int counter = 0;
            while( reader.next( value ) ) {
                reader.next(value);
                assertTrue( "Invalid Text", value.toString().compareTo(text) == 0 );
                counter++;
            }
            // Here we know that we have read half the size of data from the text
            assertEquals( "Invalid bytes read", counter , LOOP_COUNT/2 );

View Full Code Here

            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( is );
            
            // Put a boundary on half the file and just half a line, 
            // it should automaically read till end of line
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, 
                    ( ( text.length() + 1 ) * ( LOOP_COUNT/2 ) ) 
                    - (text.length()/2 ) );
            
            Text value = new Text();
            int counter = 0;
            for( int i = 0; i < LOOP_COUNT / 2; i++ ) {
                reader.next(value);
                assertEquals( "Invalid Text", value.toString().compareTo(text), 
                        0 );
                counter++;
            }
            assertEquals( "Invalid bytes read", reader.getPosition() , 
                    counter * ( text.length() + 1 ) );        
            
            testFile.deleteOnExit();
            
        } catch (IOException e) {

View Full Code Here

            
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( bzis );
            // Put a boundary on half the file and just half a line, it 
            // should automaically read till end of line
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, 
                    ( ( text.length() + 1 ) * ( LOOP_COUNT/2 ) ) 
                    - (text.length()/2 ) );
            
            Text value = new Text();
            int counter = 0;
            for( int i = 0; i < LOOP_COUNT / 2; i++ ) {
                reader.next(value);
                assertEquals( "Invalid Text", value.toString().compareTo(text), 
                        0 );
                counter++;
            }
            assertEquals( "Invalid bytes read",

View Full Code Here

            }
            ps.close();
            
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( is );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
            
            Text value = new Text();
            int counter = 0;
            while( reader.next(value) ) {
                if( counter % 2 == 0 ) {
                    assertTrue( "Invalid Text", value.toString().compareTo(text) == 0 );
                } else {
                    assertTrue( "Invalid Text", value.toString().compareTo("") == 0 );                    
                }

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.impl.io.PigLineRecordReader

org.apache.hadoop.io.LongWritable

org.apache.hadoop.mapred.LineRecordReader

org.apache.pig.builtin.PigStorage

org.apache.pig.builtin.TextLoader

org.apache.pig.test.TestPigLineRecordReader

java.io.ByteArrayOutputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.