Examples of FileSpec

flex2.compiler.FileSpec
A list of files, which are not required to follow the single public definition rule. This is used by compc's include-sources option and by Flash Builder. It's similar to ASC's include option. @author Clement Wong
org.apache.pig.impl.io.FileSpec
A simple class that specifies a file name and storage function which is used to read/write it
org.pdfclown.documents.fileSpecs.FileSpec
fanochizzolini.it) @since 0.0.7 @version 0.1.0

Examples of org.apache.pig.impl.io.FileSpec

            }
            
        }
        
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            try {
                pc.connect();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

    public static FileSpec checkLeafIsStore(
            PhysicalPlan plan,
            PigContext pigContext) throws ExecException {
        try {
            PhysicalOperator leaf = plan.getLeaves().get(0);
            FileSpec spec = null;
            if(!(leaf instanceof POStore)){
                String scope = leaf.getOperatorKey().getScope();
                POStore str = new POStore(new OperatorKey(scope,
                    NodeIdGenerator.getGenerator().getNextNodeId(scope)));
                spec = new FileSpec(FileLocalizer.getTemporaryPath(
                    pigContext).toString(),
                    new FuncSpec(Utils.getTmpFileCompressorName(pigContext)));
                str.setSFile(spec);
                plan.addAsLeaf(str);
            } else{

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

    public void visit(LOGenerate g) throws VisitorException {
        throw new VisitorException("LOGenerate is not supported.");
    }
    
    public void visit(LOLoad load) throws VisitorException{      
        FileSpec fs = load.getInputFile();
        
        LogicalSchema s = null;
        try {
            s = Util.translateSchema(load.getSchema());
        }catch(Exception e) {

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        pc.connect();
        GenPhyOp.setPc(pc);
        POLoad ld1 = GenPhyOp.topLoadOp();
        String curDir = System.getProperty("user.dir");
        String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
        FileSpec fSpec = new FileSpec("file:"+ inpDir +"passwd", new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
        ld1.setLFile(fSpec);
        
        POLoad ld2 = GenPhyOp.topLoadOp();
        ld2.setLFile(fSpec);

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec


        Map<LOUserFunc, Pair<LogicalPlan, LogicalOperator>> scalarMap = scalarFinder.getScalarMap();


        try {
            for(Map.Entry<LOUserFunc, Pair<LogicalPlan, LogicalOperator>> scalarEntry: scalarMap.entrySet()) {
                FileSpec fileSpec;
                String alias = scalarEntry.getKey().getImplicitReferencedOperator().getAlias();
                LogicalOperator store;


                LogicalPlan referredPlan = g.getAliases().get(g.getAliasOp().get(alias));


                // If referredPlan already has a store, 
                // we just use it instead of adding one from our pocket
                store = referredPlan.getLeaves().get(0);
                if(store instanceof LOStore 
                        &&
                        ((LOStore)store).getOutputFile().getFuncName().equals(
                                InterStorage.class.getName())                                            
                ) {
                        // use this store
                        fileSpec = ((LOStore)store).getOutputFile();
                }
                else {
                    // add new store
                    FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
                    fileSpec = new FileSpec(FileLocalizer.getTemporaryPath(pigContext).toString(), funcSpec);
                    store = new LOStore(referredPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)),
                            fileSpec, alias);
                    referredPlan.addAsLeaf(store);
                    ((LOStore)store).setTmpStore(true);
                    scalarEntry.getKey().setImplicitReferencedOperator(store);
                }
                lp.mergeSharedPlan(referredPlan);


                // Attach a constant operator to the ReadScalar func
                LogicalPlan innerPlan = scalarEntry.getValue().first;
                LOConst rconst = new LOConst(innerPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)), fileSpec.getFileName());
                rconst.setType(DataType.CHARARRAY);


                innerPlan.add(rconst);
                innerPlan.connect(rconst, scalarEntry.getKey());

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
        if ((System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")))
            inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
        // copy passwd file to cluster and set that as the input location for the load
        Util.copyFromLocalToCluster(cluster, inpDir + "passwd", "passwd");
        FileSpec inpFSpec = new FileSpec("passwd", new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
        POLoad ld = GenPhyOp.topLoadOp();
        ld.setLFile(inpFSpec);
        ld.setPc(pc);
        
        DataBag inpDB = DefaultBagFactory.getInstance().newDefaultBag();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec


        String pigStorage = PigStorage.class.getName() ;


        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  ConfigurationUtil.toConfiguration(new Properties())) ;


        // set schemas
        load1.setEnforcedSchema(null) ;

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec


        String pigStorage = PigStorage.class.getName() ;


        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  ConfigurationUtil.toConfiguration(new Properties())) ;


        // set schemas
        load1.setEnforcedSchema(null) ;

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

            new org.apache.pig.newplan.logical.relational.LogicalPlan();
        
        LogicalSchema aschema = new LogicalSchema();      
        aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        aschema.addField(new LogicalSchema.LogicalFieldSchema("d", null, DataType.BYTEARRAY));
        LOLoad load = new LOLoad(new FileSpec("/test/d.txt", new FuncSpec("org.apache.pig.builtin.PigStorage")), aschema, expected, null);
        expected.add(load);
        
        LOForEach foreach = new LOForEach(expected);
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan = new org.apache.pig.newplan.logical.relational.LogicalPlan();
        LOInnerLoad l1 = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(l1);
        LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(l2);
        
        List<LogicalExpressionPlan> eps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
        LogicalExpressionPlan p1 = new LogicalExpressionPlan();
        p1.add(new ProjectExpression(p1, 0, -1, gen));
        LogicalExpressionPlan p2 = new LogicalExpressionPlan();
        p2.add(new ProjectExpression(p2, 1, -1, gen));
        eps.add(p1);
        eps.add(p2);
        
        
        innerPlan.add(gen);
        innerPlan.connect(l1, gen);
        innerPlan.connect(l2, gen);
        
        foreach.setInnerPlan(innerPlan);      
        expected.add(foreach);
        
        LOStore s = new LOStore(expected, new FileSpec("/test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
      
        expected.add(s);
        
        expected.connect(load, foreach);
        expected.connect(foreach, s);

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        aschema3.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
        aschema2.addField(new LogicalSchema.LogicalFieldSchema("t", aschema3, DataType.TUPLE));
        aschema2.setTwoLevelAccessRequired(true);
        aschema.addField(new LogicalSchema.LogicalFieldSchema("d", aschema2, DataType.BAG));        
        
        LOLoad load = new LOLoad(new FileSpec("/test/d.txt", new FuncSpec("org.apache.pig.builtin.PigStorage")), aschema, expected, null);
        expected.add(load);         
        
        LOForEach foreach2 = new LOForEach(expected);
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan = new org.apache.pig.newplan.logical.relational.LogicalPlan();
        LOInnerLoad l1 = new LOInnerLoad(innerPlan, foreach2, 0);
        innerPlan.add(l1);
        LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach2, 1);
        innerPlan.add(l2);
        
        List<LogicalExpressionPlan>  eps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
        LogicalExpressionPlan p1 = new LogicalExpressionPlan();
        new ProjectExpression(p1, 0, -1, gen);
        LogicalExpressionPlan p2 = new LogicalExpressionPlan();        
        new ProjectExpression(p2, 1, -1, gen);
        eps.add(p1);
        eps.add(p2);
        
        innerPlan.add(gen);
        innerPlan.connect(l1, gen);
        innerPlan.connect(l2, gen);
        
        foreach2.setInnerPlan(innerPlan);      
        expected.add(foreach2); 
                
        LOStore s = new LOStore(expected, new FileSpec("/test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
      
        expected.add(s);
        
        expected.connect(load, foreach2);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.