Package org.apache.pig.builtin

Examples of org.apache.pig.builtin.BinStorage

340
341
342
343
344
345
346
347
348
349
350
                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
        // use local file system to get the keyDistFile
        Configuration conf = new Configuration(false);           
        conf.set(MapRedUtil.FILE_SYSTEM_NAME, "file:///");

        ReadToEndLoader loader = new ReadToEndLoader(new BinStorage(), conf,
                keyDistFile, 0);
        DataBag partitionList;
        Tuple t = loader.getNext();
        if(t==null) {
            throw new RuntimeException("Empty samples file");
View Full Code Here
322
323
324
325
326
327
328
329
330
331
332
                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
           
            // use local file system to get the quantilesFile
            Configuration conf = new Configuration(false);           
            conf.set(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
           
            ReadToEndLoader loader = new ReadToEndLoader(new BinStorage(),
                    conf, quantilesFile, 0);
            DataBag quantilesList;
            Tuple t = loader.getNext();
            if(t!=null)
            {
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
    t.appendField(weights);
    b.add(t);
   
    String fileName = "file:"+File.createTempFile("tmp", "");
    PigFile f = new PigFile(fileName);
    f.store(b, new BinStorage(), pigServer.getPigContext());
   
   
    pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName.toString()) + "' using BinStorage();");
    pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
    Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
       
        log.info("Running Store...");
        String storeFile = File.createTempFile("pig-tmp", "").getAbsolutePath();
        PigFile store = new PigFile(storeFile);
        store.store(bag, new BinStorage(), pigContext);
        log.info("Done.");

        log.info("Running Load...");
        PigFile load = new PigFile(storeFile);
        DataBag loaded = load.load(new BinStorage(), pigContext);
        log.info("Done.");

        assertTrue(bag.size() == loaded.size());

        Iterator<Tuple> it1 = bag.iterator();
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
        if (quantilesFile.length() == 0)
            throw new RuntimeException("Sort paritioner used but no quantiles found");
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
           
            Tuple t;
            ArrayList<Tuple> quantiles = new ArrayList<Tuple>();
           
            while(true){
                t = loader.getNext();
                if (t==null)
                    break;
                quantiles.add(t);
            }
            this.quantiles = quantiles.toArray(new Tuple[0]);
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    if (job != null) {
      is = FileLocalizer.openDFSFile(keyDistFile,ConfigurationUtil.toProperties(job));
    } else {
      is = FileLocalizer.openDFSFile(keyDistFile);
    }
    BinStorage loader = new BinStorage();
    DataBag partitionList;
    loader.bindTo(keyDistFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
    Tuple t = loader.getNext();
    if(t==null) {
      throw new RuntimeException("Empty samples file");
    }
    // The keydist file is structured as (key, min, max)
    // min, max being the index of the reducers
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.BinStorage

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.Path
  • org.apache.pig.backend.hadoop.datastorage.HDataStorage
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SortPartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil
  • org.apache.pig.data.Tuple
  • org.apache.pig.impl.io.BinStorageInputFormat
  • org.apache.pig.impl.io.BinStorageOutputFormat
  • org.apache.pig.impl.io.BufferedPositionedInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.