Examples of org.apache.pig.backend.datastorage.DataStorage.asElement()

Package org.apache.pig.backend.datastorage

Class org.apache.pig.backend.datastorage.DataStorage

Examples of org.apache.pig.backend.datastorage.DataStorage.asElement()

org.apache.pig.backend.datastorage.DataStorage.asElement()
Creates an entity handle for an object (no containment relation) from a String @param name of the object @return an object descriptor @throws DataStorageException if name does not conform to naming convention enforced by the Data Storage.

            storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));


            String fullPath = FileLocalizer.fullPath(loc, storage);


            if(storage.isContainer(fullPath)) {
                ElementDescriptor metaFilePath = storage.asElement(fullPath, metaname);
                if (exists(metaFilePath)) {
                    metaFileSet.add(metaFilePath);
                }
            } else {
                ElementDescriptor[] descriptors = storage.asCollection(loc);

View Full Code Here

                    } else { // descriptor instanceof HDirectory
                        container = (HDirectory)descriptor;
                    }


                    // if no custom schema, try the parent directory
                    ElementDescriptor metaFilePath = storage.asElement(container, metaname);
                    if (exists(metaFilePath)) {
                        metaFileSet.add(metaFilePath);
                    }
                }
            }

View Full Code Here

    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);

View Full Code Here

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);

View Full Code Here

                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
                    String fn;

View Full Code Here


    }


    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }


    public static long getSize(String fileName) throws IOException {
      Configuration conf = PigMapReduce.sJobConfInternal.get();

View Full Code Here

        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }


    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);


        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);


        long size = 0;

View Full Code Here

        List<ElementDescriptor> filePaths = new ArrayList<ElementDescriptor>();
        for (int m = 0; m < elems.length; m++) {
            paths.add(elems[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }

View Full Code Here

        ElementDescriptor srcElement = null;
        ElementDescriptor dstElement = null;


        try {
            srcElement = dfs.asElement(src);
            dstElement = dstStorage.asElement(dst);
        }
        catch (DataStorageException e) {
            byte errSrc = getErrorSource();
            int errCode = 0;
            switch(errSrc) {

View Full Code Here

    }


    static Path getCurrentDir(PigContext pigContext) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ContainerDescriptor desc = dfs.getActiveContainer();
        ElementDescriptor el = dfs.asElement(desc);
        return new Path(el.toString());
    }


    static void setHdfsServers(String absolutePath, PigContext pigContext) throws URISyntaxException {
        // Get native host

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.