Examples of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor


    @Test
    public void testMacroDef() throws Exception{
        log.info("Starting test testMacroDef() ...");


        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input6.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");


        String[] arg = {"date=20080228"};
        String[] argFiles = null;
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);


        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult6.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));

View Full Code Here

        log.info("Done");
    }


    @Test
    public void testCmdlineParamCurlySyntax() throws Exception{
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input7.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");


        String[] arg = {"date=20080228"};
        String[] argFiles = null;
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);


        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));

View Full Code Here

        PigServer server = new PigServer(ExecType.LOCAL);
        PigContext context = server.getPigContext();
        
        String strCmd = "A = load 'bar';\nB = foreach A generate $0;";
        
        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pin = new BufferedReader(new StringReader(strCmd));  
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile(pin, writer, null, null);
        pin = new BufferedReader(new StringReader(writer.toString()));
             
        Grunt grunt = new Grunt(pin, context);
        int results[] = grunt.exec();
        for (int i=0; i<results.length; i++) {

View Full Code Here

             }
        }


        // do parameter substitution
        try {
            ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
            StringWriter writer = new StringWriter();
            psp.genSubstitutedFile(new BufferedReader(new InputStreamReader(in)),
                                   writer,
                                   paramList.size() > 0 ? paramList.toArray(new String[0]) : null,
                                   paramsFiles!=null ? paramsFiles.toArray(new String[0]) : null);


            return writer.toString();

View Full Code Here

    for (String param: paramFiles) {
        FileLocalizer.FetchFileRet localFileRet = FileLocalizer.fetchFile(properties, param);
        paramFiles2.add(localFileRet.file.getAbsolutePath());
    }    
    
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];


    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null, 
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));


    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null, 
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}

View Full Code Here

             }
        }


        // do parameter substitution
        try {
            ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
            StringWriter writer = new StringWriter();
            psp.genSubstitutedFile(new BufferedReader(new InputStreamReader(in)),
                                   writer,
                                   paramList.size() > 0 ? paramList.toArray(new String[0]) : null,
                                   paramsFiles!=null ? paramsFiles.toArray(new String[0]) : null);


            return writer.toString();

View Full Code Here

// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(PigContext context, BufferedReader origPigScript,
                                            String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{


    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];


    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, context.getParams().size() > 0 ? context.getParams().toArray(type1) : null,
                                context.getParamFiles().size() > 0 ? context.getParamFiles().toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));


    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  context.getParams().size() > 0 ? context.getParams().toArray(type1) : null,
                                context.getParamFiles().size() > 0 ? context.getParamFiles().toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}

View Full Code Here

            for (String param : getScriptContext().getPigContext().getParams()) {
                plist.add(param);
            }
        }
        
        ParameterSubstitutionPreprocessor psp = 
            new ParameterSubstitutionPreprocessor(50);
        
        String[] params = new String[1];
        
        StringWriter writer = new StringWriter();
        BufferedReader in = new BufferedReader(new StringReader(qstr));
        String[] type1 = new String[1];
        try {
            psp.genSubstitutedFile(in, writer, plist.toArray(params), 
                    scriptContext.getPigContext().getParamFiles()!=null && 
                    scriptContext.getPigContext().getParamFiles().size() > 0 ? 
                    scriptContext.getPigContext().getParamFiles().toArray(type1) : null);
        } catch (org.apache.pig.tools.parameters.ParseException e) {
            throw new IOException("Param substitution failed", e);

View Full Code Here


    private String runPreprocessor(String script, List<String> params, 
                                   List<String> files) 
        throws IOException, ParseException {


        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();


        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)), 
                                   writer,  
                                   params.size() > 0 ? params.toArray(new String[0]) : null, 
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());

View Full Code Here

        ps.println("b = filter a by $0 == '$querystring';");
        ps.close();


        String[] arg = {"querystring='中文'"};


      ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pigIStream = new BufferedReader(new FileReader(queryFile.toString()));
        StringWriter pigOStream = new StringWriter();


        psp.genSubstitutedFile(pigIStream , pigOStream , arg, null);


        assertTrue(pigOStream.toString().contains("中文"));


        queryFile.delete();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

Example1

org.apache.pig.impl.PigContext

org.apache.pig.Main

org.apache.pig.parser.PigMacro

org.apache.pig.PigServer

org.apache.pig.pigunit.PigTest

org.apache.pig.scripting.Pig

org.apache.pig.test.TestGrunt

org.apache.pig.test.TestParamSubPreproc

org.apache.pig.test.TestUTF8

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.