Package org.apache.pig.tools.parameters

Examples of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
                                            ArrayList<String> paramFiles, String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
    PigTest.getCluster();

    BufferedReader pigIStream = new BufferedReader(new StringReader(this.originalTextPigScript));
    StringWriter pigOStream = new StringWriter();

    ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
    ps.genSubstitutedFile(pigIStream, pigOStream, args, argFiles);

    String substitutedPig = pigOStream.toString();
    LOG.info(substitutedPig);

    File f = File.createTempFile("tmp", "pigunit");
View Full Code Here
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
    private String runPreprocessor(String script, List<String> params,
                                   List<String> files)
        throws IOException, ParseException {

        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();

        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
                                   writer, 
                                   params.size() > 0 ? params.toArray(new String[0]) : null,
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());
View Full Code Here
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
    for (String param: paramFiles) {
        FileLocalizer.FetchFileRet localFileRet = FileLocalizer.fetchFile(properties, param);
        paramFiles2.add(localFileRet.file.getAbsolutePath());
    }   
   
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here
593
594
595
596
597
598
599
600
601
602
603
604
605
                    paramList.add(entry.getKey()+"="+entry.getValue());
                }
            }
           
            // do parameter substitution
            ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
            StringWriter writer = new StringWriter();
            psp.genSubstitutedFile(new BufferedReader(new InputStreamReader(new FileInputStream(fileName))),
                                   writer, 
                                   paramList.size() > 0 ? paramList.toArray(new String[0]) : null,
                                   paramsFiles!=null ? paramsFiles.toArray(new String[0]) : null);
           
            GruntParser grunt = new GruntParser(new StringReader(writer.toString()));
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
        ps.println("b = filter a by $0 == '$querystring';");
        ps.close();
     
        String[] arg = {"querystring='中文'"};
       
      ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pigIStream = new BufferedReader(new FileReader(queryFile.toString()));
        StringWriter pigOStream = new StringWriter();

        psp.genSubstitutedFile(pigIStream , pigOStream , arg, null);
       
        assertTrue(pigOStream.toString().contains("中文"));
       
        queryFile.delete();
    }
View Full Code Here
844
845
846
847
848
849
850
851
852
853
854
855
856
857
        PigServer server = new PigServer(ExecType.LOCAL);
        PigContext context = server.getPigContext();
       
        String strCmd = "A = load 'bar';\nB = foreach A generate $0;";
       
        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pin = new BufferedReader(new StringReader(strCmd)); 
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile(pin, writer, null, null);
        pin = new BufferedReader(new StringReader(writer.toString()));
            
        Grunt grunt = new Grunt(pin, context);
        int results[] = grunt.exec();
        for (int i=0; i<results.length; i++) {
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
        ps.println("b = filter a by $0 == '$querystring';");
        ps.close();
     
        String[] arg = {"querystring='中文'"};
       
      ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pigIStream = new BufferedReader(new FileReader(queryFile.toString()));
        StringWriter pigOStream = new StringWriter();

        psp.genSubstitutedFile(pigIStream , pigOStream , arg, null);
       
        assertTrue(pigOStream.toString().contains("中文"));
       
        queryFile.delete();
    }
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
    private String runPreprocessor(String script, List<String> params,
                                   List<String> files)
        throws IOException, ParseException {

        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();

        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
                                   writer, 
                                   params.size() > 0 ? params.toArray(new String[0]) : null,
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());
View Full Code Here
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
                                            ArrayList<String> paramFiles, String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

  • Example1
  • org.apache.pig.impl.PigContext
  • org.apache.pig.Main
  • org.apache.pig.parser.PigMacro
  • org.apache.pig.PigServer
  • org.apache.pig.pigunit.PigTest
  • org.apache.pig.scripting.Pig
  • org.apache.pig.test.TestGrunt
  • org.apache.pig.test.TestParamSubPreproc
  • org.apache.pig.test.TestUTF8
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.