Examples of PigStorage

org.apache.pig.builtin.PigStorage
A load function that parses a line of input into fields using a delimiter to set the fields. The delimiter is given as a regular expression. See {@link java.lang.String#split(String)} and {@link java.util.regex.Pattern}for more information.

Examples of org.apache.pig.builtin.PigStorage

        String output = "/pig/out";
        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
        
        InputStream op = FileLocalizer.open(output, pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE); 
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }


        // Run the query and check the results
        Util.checkQueryOutputs(outputs.iterator(), expectedResults);

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
        
        InputStream op = FileLocalizer.open(output+"/bar", 
                                            pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE); 
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }


        // Run the query and check the results
        Util.checkQueryOutputs(outputs.iterator(), expectedResults);

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
        
        InputStream op = FileLocalizer.open(output+"/foobar", 
                                            pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE); 
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }


        // Run the query and check the results
        Util.checkQueryOutputs(outputs.iterator(), expectedResults);

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

                if (success && !descriptor.exists()) {
                        
                    // create an empty output file
                    PigFile f = new PigFile(outputFile.toString(), false);
                    f.store(BagFactory.getInstance().newDefaultBag(), 
                            new PigStorage(), 
                            pom.pigContext);
                }
            }
            catch (DataStorageException e) {
                throw WrappedIOException.wrap("Failed to obtain descriptor for " + outputFile.toString(), e);

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    public PigRecordWriter getRecordWriter(FileSystem fs, JobConf job, Path outputDir, String name, Progressable progress)
            throws IOException {
        StoreFunc store;
        String storeFunc = (String) ObjectSerializer.deserialize(job.get("pig.storeFunc", "")) ;
        if (storeFunc.length() == 0) {
            store = new PigStorage();
        } else {
            store = (StoreFunc) PigContext.instantiateFuncFromSpec(storeFunc);
        }


        String parentName = FileOutputFormat.getOutputPath(job).getName();

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage


  @Override
  public void initialize(JobContext context, Properties storageDriverArgs) throws IOException {


    lf = storageDriverArgs.containsKey(delim) ?
        new PigStorage(storageDriverArgs.getProperty(delim)) : new PigStorage();
    super.initialize(context, storageDriverArgs);
  }

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    }


    @SuppressWarnings("unchecked")
  public void init(DataStorage base) throws IOException {
        if (parser == null) {
          loader = new PigStorage();
        } else {
            try {
                loader = (LoadFunc) PigContext.instantiateFuncFromSpec(parser);
            } catch (Exception exp) {
                int errCode = 2081;

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    public static StoreFunc getStoreFunc(JobConf conf) throws ExecException {
        StoreFunc store;
        try {
            String storeFunc = conf.get("pig.storeFunc", "");
            if (storeFunc.length() == 0) {
                store = new PigStorage();
            } else {
                storeFunc = (String) ObjectSerializer.deserialize(storeFunc);
                store = (StoreFunc) PigContext
                        .instantiateFuncFromSpec(storeFunc);
            }

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

        inpDB = GenRandomData.genRandFullTupTextDataBag(new Random(), 10, 100);
        Tuple t = new DefaultTuple();
        t.append(inpDB);
        proj.attachInput(t);
        assertTrue(store() != null);
        PigStorage ps = new PigStorage(":");
        
        int size = 0;
        BufferedReader br = new BufferedReader(new FileReader("/tmp/storeTest.txt"));
        for(String line=br.readLine();line!=null;line=br.readLine()){
            String[] flds = line.split(":",-1);
            t = new DefaultTuple();
            t.append(flds[0].compareTo("")!=0 ? ps.bytesToBag(flds[0].getBytes()) : null);
            t.append(flds[1].compareTo("")!=0 ? ps.bytesToCharArray(flds[1].getBytes()) : null);
            t.append(flds[2].compareTo("")!=0 ? ps.bytesToCharArray(flds[2].getBytes()) : null);
            t.append(flds[3].compareTo("")!=0 ? ps.bytesToDouble(flds[3].getBytes()) : null);
            t.append(flds[4].compareTo("")!=0 ? ps.bytesToFloat(flds[4].getBytes()) : null);
            t.append(flds[5].compareTo("")!=0 ? ps.bytesToInteger(flds[5].getBytes()) : null);
            t.append(flds[6].compareTo("")!=0 ? ps.bytesToLong(flds[6].getBytes()) : null);
            t.append(flds[7].compareTo("")!=0 ? ps.bytesToMap(flds[7].getBytes()) : null);
            t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null);
            
            assertEquals(true, TestHelper.bagContains(inpDB, t));
            ++size;
        }
        assertEquals(true, size==inpDB.size());

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

        inpDB.add(inputTuple);
        Tuple t = new DefaultTuple();
        t.append(inpDB);
        proj.attachInput(t);
        assertTrue(store() != null);
        PigStorage ps = new PigStorage(":");
        
        int size = 0;
        BufferedReader br = new BufferedReader(new FileReader("/tmp/storeTest.txt"));
        for(String line=br.readLine();line!=null;line=br.readLine()){
            System.err.println("Complex data: ");
            System.err.println(line);
            String[] flds = line.split(":",-1);
            t = new DefaultTuple();
            t.append(flds[0].compareTo("")!=0 ? ps.bytesToBag(flds[0].getBytes()) : null);
            t.append(flds[1].compareTo("")!=0 ? ps.bytesToCharArray(flds[1].getBytes()) : null);
            t.append(flds[2].compareTo("")!=0 ? ps.bytesToCharArray(flds[2].getBytes()) : null);
            t.append(flds[3].compareTo("")!=0 ? ps.bytesToDouble(flds[3].getBytes()) : null);
            t.append(flds[4].compareTo("")!=0 ? ps.bytesToFloat(flds[4].getBytes()) : null);
            t.append(flds[5].compareTo("")!=0 ? ps.bytesToInteger(flds[5].getBytes()) : null);
            t.append(flds[6].compareTo("")!=0 ? ps.bytesToLong(flds[6].getBytes()) : null);
            t.append(flds[7].compareTo("")!=0 ? ps.bytesToMap(flds[7].getBytes()) : null);
            t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null);
            t.append(flds[9].compareTo("")!=0 ? ps.bytesToCharArray(flds[9].getBytes()) : null);
            
            assertTrue(TestHelper.tupleEquals(inputTuple, t));
            ++size;
        }
        FileLocalizer.delete(fSpec.getFileName(), pc);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.