Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage.asElement()


        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
View Full Code Here


                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
View Full Code Here

                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here

   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here

        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
View Full Code Here

                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
View Full Code Here

                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here

   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.