Package org.apache.pig.backend.hadoop.datastorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage


     * @param conf      configuration object
     * @return Set of element descriptors for all metadata files associated with the files on the path.
     */
    protected Set<ElementDescriptor> findMetaFile(String path, String prefix, Configuration conf)
        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
            for(ElementDescriptor descriptor : descriptors) {
                String fileName = null, parentName = null;
                ContainerDescriptor parentContainer = null;
                if (descriptor instanceof HFile) {
                    Path descriptorPath = ((HFile) descriptor).getPath();
                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here


    // Implementation of StoreMetaData interface
   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here

            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);
       
           
View Full Code Here

            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);
       
           
View Full Code Here

     * @param conf      configuration object
     * @return Set of element descriptors for all metadata files associated with the files on the path.
     */
    protected Set<ElementDescriptor> findMetaFile(String path, String prefix, Configuration conf)
        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
            for(ElementDescriptor descriptor : descriptors) {
                String fileName = null, parentName = null;
                ContainerDescriptor parentContainer = null;
                if (descriptor instanceof HFile) {
                    Path descriptorPath = ((HFile) descriptor).getPath();
                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here

    // Implementation of StoreMetaData interface
   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here

        Configuration conf = job.getConfiguration();
        Properties props = ConfigurationUtil.toProperties(conf);
        // since local mode now is implemented as hadoop's local mode
        // we can treat either local or hadoop mode as hadoop mode - hence
        // we can use HDataStorage and FileLocalizer.openDFSFile below
        HDataStorage storage = new HDataStorage(props);
       
        // At compile time in batch mode, the file may not exist
        // (such as intermediate file). Just return null - the
        // same way as we would if we did not get a valid record
        String[] locations = getPathStrings(location);
View Full Code Here

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        properties);
            }
            break;
           
            default:
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.datastorage.HDataStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.