Examples of org.apache.pig.backend.datastorage.ElementDescriptor

Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.ElementDescriptor

org.apache.pig.backend.datastorage.ElementDescriptor
DataStorageElementDescriptor provides methods necessary to manage an element in a DataStorage.

    @Override
    protected void processRemove(String path, String options ) throws IOException
    {
        if(mExplain == null) { // process only if not in "explain" mode


            ElementDescriptor dfsPath = mDfs.asElement(path);    
            executeBatch();
        
            if (!dfsPath.exists()) {
                if (options == null || !options.equalsIgnoreCase("force")) {
                    throw new IOException("File or directory " + path + " does not exist."); 
                }
            }
            else {
                
                dfsPath.delete();
            }
        } else {
            log.warn("'rm/rmf' statement is ignored while processing 'explain -script' or '-check'");
        }
    }

View Full Code Here

     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);


        return length * replication;

View Full Code Here

     * @param filename to test
     * @return true if file exists, false otherwise
     * @throws IOException
     */
    public boolean existsFile(String filename) throws IOException {
        ElementDescriptor elem = pigContext.getDfs().asElement(filename);
        return elem.exists();
    }

View Full Code Here

     * @param filename to delete
     * @return true
     * @throws IOException
     */
    public boolean deleteFile(String filename) throws IOException {
        ElementDescriptor elem = pigContext.getDfs().asElement(filename);
        elem.delete();
        return true;
    }

View Full Code Here

        Collection<String> allPaths = new ArrayList<String>();
        ContainerDescriptor container = pigContext.getDfs().asContainer(dir);
        Iterator<ElementDescriptor> iter = container.iterator();
            
        while (iter.hasNext()) {
            ElementDescriptor elem = iter.next();
            allPaths.add(elem.toString());
        }
            
        String[] type = new String[1];
        return allPaths.toArray(type);
    }

View Full Code Here

     * @param filename
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);


        return length * replication;

View Full Code Here


        return length * replication;
    }
    
    public boolean existsFile(String filename) throws IOException {
        ElementDescriptor elem = pigContext.getDfs().asElement(filename);
        return elem.exists();
    }

View Full Code Here

        ElementDescriptor elem = pigContext.getDfs().asElement(filename);
        return elem.exists();
    }
    
    public boolean deleteFile(String filename) throws IOException {
        ElementDescriptor elem = pigContext.getDfs().asElement(filename);
        elem.delete();
        return true;
    }

View Full Code Here

        Collection<String> allPaths = new ArrayList<String>();
        ContainerDescriptor container = pigContext.getDfs().asContainer(dir);
        Iterator<ElementDescriptor> iter = container.iterator();
            
        while (iter.hasNext()) {
            ElementDescriptor elem = iter.next();
            allPaths.add(elem.toString());
        }
            
        return (String[])(allPaths.toArray());
    }

View Full Code Here

        ElementDescriptor[] globPaths = store.asCollection(location);
        for (int m = 0; m < globPaths.length; m++) {
            paths.add(globPaths[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }
            if (fullPath instanceof ContainerDescriptor) {
                for (ElementDescriptor child : ((ContainerDescriptor) fullPath)) {
                    paths.add(child);
                }
                continue;
            }
            Map<String, Object> stats = fullPath.getStatistics();
            long bs = (Long) (stats.get(ElementDescriptor.BLOCK_SIZE_KEY));
            long size = (Long) (stats.get(ElementDescriptor.LENGTH_KEY));
            long pos = 0;
            String name = fullPath.toString();
            // System.out.println(size + " " + name);
            if (name.endsWith(".gz") || !splittable) {
                // Anything that ends with a ".gz" we must process as a complete
                // file
                slices.add(new PigSlice(name, funcSpec, 0, size));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.backend.datastorage.ElementDescriptor

com.packetloop.packetpig.storage.JsonMetadata

org.apache.pig.backend.executionengine.PigSlicer

org.apache.pig.backend.hadoop.executionengine.mapreduceExec.MapReduceLauncher

org.apache.pig.backend.local.datastorage.LocalDir

org.apache.pig.builtin.JsonMetadata

org.apache.pig.impl.io.FileLocalizer

org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor

org.apache.pig.impl.PigContext

org.apache.pig.parser.QueryParserUtils

org.apache.pig.piggybank.storage.JsonMetadata

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.