Examples of HDataStorage

org.apache.pig.backend.hadoop.datastorage.HDataStorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        }


        LOG.info("Connecting to hadoop file system at: "
                + (nameNode == null ? LOCAL : nameNode));
        // constructor sets DEFAULT_REPLICATION_FACTOR_KEY
        ds = new HDataStorage(properties);


        if (cluster != null && !cluster.equalsIgnoreCase(LOCAL)) {
            LOG.info("Connecting to map-reduce job tracker at: "
                    + jc.get(MRConfiguration.JOB_TRACKER));
        }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

    }


    public void connect() throws ExecException {
        executionEngine.init();
        dfs = executionEngine.getDataStorage();
        lfs = new HDataStorage(URI.create("file:///"), properties);
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));


    }


    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage


        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }


    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);


        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);


        long size = 0;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

                properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
            }
        }
     
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
                
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        // So we need to reconstruct the configuration object for the non HOD case
        // In the HOD case, this is the first time the configuration object will be created
        configuration = ConfigurationUtil.toConfiguration(properties);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        }


        LOG.info("Connecting to hadoop file system at: "
                + (nameNode == null ? LOCAL : nameNode));
        // constructor sets DEFAULT_REPLICATION_FACTOR_KEY
        ds = new HDataStorage(properties);


        if (cluster != null && !cluster.equalsIgnoreCase(LOCAL)) {
            LOG.info("Connecting to map-reduce job tracker at: "
                    + jc.get(JOB_TRACKER_LOCATION));
        }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        String[] locations = LoadFunc.getPathStrings(path);
        for (String loc : locations) {
            DataStorage storage;


            storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));


            String fullPath = FileLocalizer.fullPath(loc, storage);


            if(storage.isContainer(fullPath)) {
                ElementDescriptor metaFilePath = storage.asElement(fullPath, metaname);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

    // Implementation of StoreMetaData interface


    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

    }


    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
                    String fn;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

        String[] locations = getPathStrings(location);
        for (String loc : locations) {
            // since local mode now is implemented as hadoop's local mode
            // we can treat either local or hadoop mode as hadoop mode - hence
            // we can use HDataStorage and FileLocalizer.openDFSFile below
            HDataStorage storage;
            try {
              storage = new HDataStorage((new org.apache.hadoop.fs.Path(loc)).toUri(), props);
            } catch (RuntimeException e) {
                throw new IOException(e);
            }
            if (!FileLocalizer.fileExists(loc, storage)) {
                return null;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.