Package org.apache.pig

Examples of org.apache.pig.PigServer$Graph


  protected PigServer createPigInstance() throws Exception {
    final PigContext ctx = (pigContext != null ? pigContext : new PigContext());

    // apparently if not connected, pig can cause all kind of errors
    PigServer pigServer = null;

    try {
      if (StringUtils.hasText(user)) {
        UserGroupInformation ugi = UserGroupInformation.createProxyUser(user,
            UserGroupInformation.getLoginUser());
        pigServer = ugi.doAs(new PrivilegedExceptionAction<PigServer>() {
          @Override
          public PigServer run() throws Exception {
            return new PigServer(ctx, true);
          }
        });
      }
      else {
        pigServer = new PigServer(ctx, true);
      }
    } catch (ExecException ex) {
      throw PigUtils.convert(ex);
    }


    if (!CollectionUtils.isEmpty(pathToSkip)) {
      for (String path : pathToSkip) {
        pigServer.addPathToSkip(path);
      }
    }

    if (parallelism != null) {
      pigServer.setDefaultParallel(parallelism);
    }

    if (StringUtils.hasText(jobName)) {
      pigServer.setJobName(jobName);
    }
    else {
      if (StringUtils.hasText(beanName)) {
        pigServer.setJobName(beanName);
      }
    }

    if (StringUtils.hasText(jobPriority)) {
      pigServer.setJobPriority(jobPriority);
    }

    if (validateEachStatement != null) {
      PigUtils.validateEachStatement(pigServer, validateEachStatement);
    }
View Full Code Here


   * @throws DataAccessException exception
   */
  @Override
  public <T> T execute(PigCallback<T> action) throws DataAccessException {
    Assert.notNull(action, "a valid callback is required");
    PigServer pig = createPigServer();

    try {
      // make sure pig is connected
      pig.getPigContext().connect();
      return action.doInPig(pig);

    } catch (ExecException ex) {
      throw convertPigAccessException(ex);
    } catch (IOException ex) {
      throw convertPigAccessException(ex);
    } finally {
      pig.shutdown();
    }
  }
View Full Code Here

    private PigServer pigServer;
    private static MiniCluster cluster = MiniCluster.buildCluster();
    private File tmpFile;
   
    public TestFRJoin() throws ExecException, IOException{
        pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
//        pigServer = new PigServer(ExecType.LOCAL);
       
    }
View Full Code Here

    Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
    Map<String, String> fileNameMap = new HashMap<String, String>();

    @Before
    public void setUp() throws Exception {
        pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        String origPath = FileLocalizer.fullPath("originput", pig.getPigContext());
        if (FileLocalizer.fileExists(origPath, pig.getPigContext())) {
            FileLocalizer.delete(origPath, pig.getPigContext());
        }
        Util.createInputFile(cluster, "originput",
View Full Code Here

       
        Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class);
        pigStorageWithTraceLogger.setLevel(Level.INFO);
        pigStorageWithTraceLogger.addAppender(appender);
       
        pigServer = new PigServer("local");
        //pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        tmpFile1 = File.createTempFile("prune", "txt");
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile1));
        ps.println("1\t2\t3");
        ps.println("2\t5\t2");
View Full Code Here

    }
*/
   
    @Test
    public void testDefine() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "define myudf org.apache.pig.builtin.AVG();\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
View Full Code Here

        assertTrue(null != context.getFuncSpecFromAlias("myudf"));
    }

    @Test
    public void testBagSchema() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "a = load 'input1' as (b: bag{t(i: int, c:chararray, f: float)});\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
View Full Code Here

        grunt.exec();
    }

    @Test
    public void testBagSchemaFail() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "a = load 'input1'as (b: bag{t(i: int, c:chararray, f: float)});\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
View Full Code Here

        }
    }

    @Test
    public void testBagConstant() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1', 0.4f),(2, '2', 0.45)};\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
View Full Code Here

        grunt.exec();
    }

    @Test
    public void testBagConstantWithSchema() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "a = load 'input1'; b = foreach a generate "
                + "{(1, '1', 0.4f),(2, '2', 0.45)} as "
                + "b: bag{t(i: int, c:chararray, d: double)};\n";
       
View Full Code Here

TOP

Related Classes of org.apache.pig.PigServer$Graph

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.