Package org.apache.pig.builtin

Examples of org.apache.pig.builtin.BinStorage


                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here


        // use local file system to get the keyDistFile
        Configuration conf = new Configuration(false);           
        conf.set(MapRedUtil.FILE_SYSTEM_NAME, "file:///");

        ReadToEndLoader loader = new ReadToEndLoader(new BinStorage(), conf,
                keyDistFile, 0);
        DataBag partitionList;
        Tuple t = loader.getNext();
        if(t==null) {
            throw new RuntimeException("Empty samples file");
View Full Code Here

                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here

           
            // use local file system to get the quantilesFile
            Configuration conf = new Configuration(false);           
            conf.set(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
           
            ReadToEndLoader loader = new ReadToEndLoader(new BinStorage(),
                    conf, quantilesFile, 0);
            DataBag quantilesList;
            Tuple t = loader.getNext();
            if(t!=null)
            {
View Full Code Here

    t.appendField(weights);
    b.add(t);
   
    String fileName = "file:"+File.createTempFile("tmp", "");
    PigFile f = new PigFile(fileName);
    f.store(b, new BinStorage(), pigServer.getPigContext());
   
   
    pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName.toString()) + "' using BinStorage();");
    pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
    Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here

        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
       
        log.info("Running Store...");
        String storeFile = File.createTempFile("pig-tmp", "").getAbsolutePath();
        PigFile store = new PigFile(storeFile);
        store.store(bag, new BinStorage(), pigContext);
        log.info("Done.");

        log.info("Running Load...");
        PigFile load = new PigFile(storeFile);
        DataBag loaded = load.load(new BinStorage(), pigContext);
        log.info("Done.");

        assertTrue(bag.size() == loaded.size());

        Iterator<Tuple> it1 = bag.iterator();
View Full Code Here

        if (quantilesFile.length() == 0)
            throw new RuntimeException("Sort paritioner used but no quantiles found");
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
           
            Tuple t;
            ArrayList<Tuple> quantiles = new ArrayList<Tuple>();
           
            while(true){
                t = loader.getNext();
                if (t==null)
                    break;
                quantiles.add(t);
            }
            this.quantiles = quantiles.toArray(new Tuple[0]);
View Full Code Here

    if (job != null) {
      is = FileLocalizer.openDFSFile(keyDistFile,ConfigurationUtil.toProperties(job));
    } else {
      is = FileLocalizer.openDFSFile(keyDistFile);
    }
    BinStorage loader = new BinStorage();
    DataBag partitionList;
    loader.bindTo(keyDistFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
    Tuple t = loader.getNext();
    if(t==null) {
      throw new RuntimeException("Empty samples file");
    }
    // The keydist file is structured as (key, min, max)
    // min, max being the index of the reducers
View Full Code Here

        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here

        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here

TOP

Related Classes of org.apache.pig.builtin.BinStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.