Examples of HDataStorage

org.apache.pig.backend.hadoop.datastorage.HDataStorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

                properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
            }
        }
     
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
                
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        // So we need to reconstruct the configuration object for the non HOD case
        // In the HOD case, this is the first time the configuration object will be created
        configuration = ConfigurationUtil.toConfiguration(properties);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        return lastConf;
    }


    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;        
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        wrapped.init(store);
        
        // Mimic org.apache.hadoop.mapred.FileSplit if feasible...
        String[] locations = wrapped.getLocations();
        if (locations.length > 0) {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        }


        FileSystem fs = FileSystem.get(job);
        List<SliceWrapper> splits = new ArrayList<SliceWrapper>();
        for (int i = 0; i < inputs.size(); i++) {
            DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
            ValidatingInputFileSpec spec;
            if (inputs.get(i) instanceof ValidatingInputFileSpec) {
                spec = (ValidatingInputFileSpec) inputs.get(i);
            } else {
                spec = new ValidatingInputFileSpec(inputs.get(i), store);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
     
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
                
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage


                executionEngine.init();
                
                dfs = executionEngine.getDataStorage();
                
                lfs = new HDataStorage(URI.create("file:///"),
                                        properties); 
            }
            break;
            
            default:

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

    public void connect() throws ExecException {


        switch (execType) {
            case LOCAL:
            {
                lfs = new HDataStorage(URI.create("file:///"),
                                       new Properties());
                
                dfs = lfs;
                executionEngine = new LocalExecutionEngine(this);
            }
            break;


            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);


                executionEngine.init();
                
                dfs = executionEngine.getDataStorage();
                
                lfs = new HDataStorage(URI.create("file:///"),
                                        new Properties());                
            }
            break;
            
            default:

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));


    }


    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage


        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
    
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
       
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
        
        long size = 0;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                }
        
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        return lastConf;
    }


    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;        
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        // if the execution is against Mapred DFS, set
        // working dir to /user/<userid>
        if(execType == ExecType.MAPREDUCE)
            store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
        wrapped.init(store);
        
        job.set("map.target.ops", ObjectSerializer.serialize(targetOps));
        // Mimic org.apache.hadoop.mapred.FileSplit if feasible...

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.