Examples of org.apache.pig.LoadFunc$RequiredField

org.apache.pig.LoadFunc
This interface is used to implement functions to parse records from a dataset. This also includes functions to cast raw byte data into various datatypes. These are external functions because we want loaders, whenever possible, to delay casting of datatypes until the last possible moment (i.e. don't do it on load). This means we need to expose the functionality so that other sections of the code can call back to the loader to do the cast.

  }
  
  @Test
  public void testByteArrayToOther() throws PlanException, ExecException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      assertEquals(null, res.result);
    }
  }
  
  private PhysicalPlan constructPlan(POCast op) throws PlanException {
        LoadFunc load = new TestLoader();
        op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj);
        plan.add(op);
        plan.connect(prj, op);

View Full Code Here

      t.append(r.nextInt());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextLong());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextFloat());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextDouble());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

  }
  
  @Test
  public void testStringToOther() throws PlanException, ExecException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

  }
  
  @Test
  public void testByteArrayToOther() throws PlanException, ExecException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      assertEquals(null, res.result);
    }
  }
  
  private PhysicalPlan constructPlan(POCast op) throws PlanException {
        LoadFunc load = new TestLoader();
        op.setLoadFSpec(new FuncSpec(load.getClass().getName()));
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj);
        plan.add(op);
        plan.connect(prj, op);

View Full Code Here

            joinOp.setupRightPipeline(rightPipelinePlan);
      rightMROpr.requestedParallelism = 1; // we need exactly one reducer for indexing job.        
            
            // At this point, we must be operating on map plan of right input and it would contain nothing else other then a POLoad.
            POLoad rightLoader = (POLoad)rightMROpr.mapPlan.getRoots().get(0);            
            LoadFunc rightLoadFunc = (LoadFunc) PigContext.instantiateFuncFromSpec(rightLoader.getLFile().getFuncSpec());
            joinOp.setSignature(rightLoader.getSignature());
            if(rightLoadFunc instanceof IndexableLoadFunc) {
                joinOp.setRightLoaderFuncSpec(rightLoader.getLFile().getFuncSpec());
                joinOp.setRightInputFileName(rightLoader.getLFile().getFileName());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.LoadFunc$RequiredField

com.netflix.lipstick.util.OutputSampler

com.twitter.elephantbird.pig.load.MultiFormatLoader

com.twitter.elephantbird.pig.load.TestLocationAsTuple

org.apache.hadoop.fs.Path

org.apache.hadoop.mapreduce.Counter

org.apache.pig.backend.hadoop.executionengine.HJob

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.