Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.InterStorage

64
65
66
67
68
69
70
71
72
73
74
                if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
                    conf.set("mapreduce.job.credentials.binary",
                            System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
                }
                loader = new ReadToEndLoader(
                        new InterStorage(), conf, scalarfilename, 0);
            } catch (Exception e) {
                throw new ExecException("Failed to open file '" + scalarfilename
                        + "'; error = " + e.getMessage());
            }
            try {
View Full Code Here
158
159
160
161
162
163
164
            return InterStorage.class.getName();
    }

    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }
View Full Code Here
158
159
160
161
162
163
164
            return InterStorage.class.getName();
    }

    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }
View Full Code Here
243
244
245
246
247
248
249
            return InterStorage.class.getName();
    }

    public static FileInputLoadFunc getTmpFileStorageObject(Configuration conf) throws IOException {
        boolean tmpFileCompression = conf.getBoolean("pig.tmpfilecompression", false);
        return tmpFileCompression ? new TFileStorage() : new InterStorage();
    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
                if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
                    conf.set(MRConfiguration.JOB_CREDENTIALS_BINARY,
                            System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
                }
                loader = new ReadToEndLoader(
                        new InterStorage(), conf, scalarfilename, 0);
            } catch (Exception e) {
                throw new ExecException("Failed to open file '" + scalarfilename
                        + "'; error = " + e.getMessage());
            }
            try {
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
                if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
                    conf.set("mapreduce.job.credentials.binary",
                            System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
                }
                loader = new ReadToEndLoader(
                        new InterStorage(), conf, scalarfilename, 0);
            } catch (Exception e) {
                throw new ExecException("Failed to open file '" + scalarfilename
                        + "'; error = " + e.getMessage());
            }
            try {
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    @Test
    public void getSampleOutputAllOutputLessThanMaxBytesAndRows() throws Exception {
        final String output = "testRecord";
        final String schema = "{blah: chararray}";

        final InterStorage is = getInterStorage(Lists.newArrayList(output));
        List<SampleOutput> outputs = getSampleOutputs(is, schema);

        Assert.assertEquals(outputs.get(0).getOutput(), output);
        Assert.assertEquals(outputs.get(0).getSchema(), schema);
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    public void getSampleOutputAllOutputMoreThanMaxBytes() throws Exception {
        final String schema = "{blah: chararray}";
        int recordCount = 1000;
        int byteCount = 256;

        final InterStorage is = getInterStorage(createRecords(recordCount));
        List<SampleOutput> outputs = getSampleOutputs(is, schema, recordCount, byteCount);

        Assert.assertTrue(outputs.get(0).getOutput().split("\n").length < recordCount);
        Assert.assertTrue(outputs.get(0).getOutput().length() <= byteCount);
        Assert.assertEquals(outputs.get(0).getSchema(), schema);
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    public void getSampleOutputAllOutputMoreThanMaxRows() throws Exception {
        final String schema = "{blah: chararray}";
        int recordCount = 10;
        int byteCount = 1024;

        final InterStorage is = getInterStorage(createRecords(recordCount));
        List<SampleOutput> outputs = getSampleOutputs(is, schema, recordCount, byteCount);

        Assert.assertEquals(outputs.get(0).getOutput().split("\n").length, recordCount);
        Assert.assertTrue(outputs.get(0).getOutput().length() <= byteCount);
        Assert.assertEquals(outputs.get(0).getSchema(), schema);
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
        final String output = "This record is larger than our byte count and we expect no output";
        final String schema = "{blah: chararray}";
        int recordCount = 10;
        int byteCount = 20;

        final InterStorage is = getInterStorage(Lists.newArrayList(output));
        List<SampleOutput> outputs = getSampleOutputs(is, schema, recordCount, byteCount);

        Assert.assertEquals(outputs.get(0).getOutput().length(), 0);
        Assert.assertEquals(outputs.get(0).getSchema(), schema);
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.InterStorage

  • com.netflix.lipstick.util.OutputSamplerTest
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataOutputStream
  • org.apache.hadoop.fs.Path
  • org.apache.pig.impl.builtin.ReadScalars
  • org.apache.pig.impl.util.Utils
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.