Examples of org.apache.pig.impl.io.InterStorage

org.apache.pig.impl.io.InterStorage
LOAD FUNCTION FOR PIG INTERNAL USE ONLY! This load function is used for storing intermediate data between MR jobs of a pig query. The serialization format of this load function can change in newer versions of pig, so this should NOT be used to store any persistent data.

        when(pos.getSchema()).thenReturn(new Schema(new Schema.FieldSchema("blah", (byte) 55)));
        return pos;
    }


    private InterStorage getInterStorage(List<String> returnRecords) throws IOException {
        InterStorage is = mock(InterStorage.class);
        when(is.getInputFormat()).thenReturn(new InterInputFormat());


        List<Tuple> tuples = new LinkedList<Tuple>();
        for (String record : returnRecords) {
            Tuple tup = mock(Tuple.class);
            when(tup.toDelimitedString(OutputSampler.DELIMITER)).thenReturn(record);
            tuples.add(tup);
        }
        tuples.add(null);
        when(is.getNext()).thenReturn(tuples.remove(0), tuples.toArray(new Tuple[0]));


        return is;
    }

View Full Code Here

            return InterStorage.class.getName();
    }


    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }

View Full Code Here

                if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
                    conf.set("mapreduce.job.credentials.binary", 
                            System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
                }
                loader = new ReadToEndLoader(
                        new InterStorage(), conf, scalarfilename, 0);
            } catch (Exception e) {
                throw new ExecException("Failed to open file '" + scalarfilename
                        + "'; error = " + e.getMessage());
            }
            try {

View Full Code Here

            return InterStorage.class.getName();
    }


    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }

View Full Code Here

            return InterStorage.class.getName();
    }


    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.impl.io.InterStorage

com.netflix.lipstick.util.OutputSamplerTest

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.Path

org.apache.pig.impl.builtin.ReadScalars

org.apache.pig.impl.util.Utils

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.