Package org.apache.pig.LoadPushDown

Examples of org.apache.pig.LoadPushDown.RequiredFieldResponse

326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        return result;
    }
   
    public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException
    {
        RequiredFieldResponse response = new RequiredFieldResponse(false);
        if (mSchema == null)
            return response;
       
        if (requiredFieldList.getFields() == null)
            return response;
       
        if (requiredFieldList.getFields()==null)
        {
            return response;
        }
       
        this.requiredFieldList = requiredFieldList;
        if(mLoadFunc instanceof LoadPushDown) {
            response = ((LoadPushDown)mLoadFunc).pushProjection(requiredFieldList);
        } else {
            // loadfunc does not support pushing projections
            response = new RequiredFieldResponse(false);
        }
        if (!response.getRequiredFieldResponse())
            return response;
       
        // Change LOLoad schema to reflect this pruning
        TreeSet<Integer> prunedIndexSet = new TreeSet<Integer>();
        for (int i=0;i<mSchema.size();i++)
            prunedIndexSet.add(i);

        for (int i=0;i<requiredFieldList.getFields().size();i++)
        {
            RequiredField requiredField = requiredFieldList.getFields().get(i);
            if (requiredField.getIndex()>=0)
                prunedIndexSet.remove(requiredField.getIndex());
            else
            {
               
                try {
                    int index = mSchema.getPosition(requiredField.getAlias());
                    if (index>0)
                        prunedIndexSet.remove(index);
                } catch (FrontendException e) {
                    return new RequiredFieldResponse(false);
                }
               
            }
        }
       
View Full Code Here
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
                j++;
            requiredFieldList.getFields().add(j, requiredField);
        }
       
        boolean[] columnRequired = new boolean[load.getSchema().size()];
        RequiredFieldResponse response = null;
        try {
            response = load.pushProjection(requiredFieldList);   
           
        } catch (FrontendException e) {
            log.warn("fieldsToRead on "+load+" throw an exception, skip it");
        }
       
        // If the request is not granted, probably the loader support position prune only,
        // and do not prune map key pruning (such as PigStorage). Drop all map keys (means
        // we do not prune map keys) and try again
        if (response==null || !response.getRequiredFieldResponse())
        {
            for (RequiredField rf : requiredFieldList.getFields())
            {
                if (rf.getType() == DataType.MAP)
                    rf.setSubFields(null);
            }
            try {
                response = load.pushProjection(requiredFieldList);   
            } catch (FrontendException e) {
                log.warn("fieldsToRead on "+load+" throw an exception, skip it");
            }
        }
       
        // Loader does not support column pruning, insert foreach
        LogicalOperator forEach = null;
        if (response==null || !response.getRequiredFieldResponse())
        {
            List<Integer> columnsToProject = new ArrayList<Integer>();
            for (RequiredField rf : requiredFieldList.getFields())
                columnsToProject.add(rf.getIndex());
           
View Full Code Here
2011
2012
2013
2014
2015
2016
2017
            try {
                UDFContext.getUDFContext().getUDFProperties(this.getClass()).setProperty(signature, ObjectSerializer.serialize(aliases));
            } catch (IOException e) {
                throw new FrontendException(e);
            }
            return new RequiredFieldResponse(true);
        }
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
        return result;
    }
   
    public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException
    {
        RequiredFieldResponse response = new RequiredFieldResponse(false);
        if (mSchema == null)
            return response;
       
        if (requiredFieldList.getFields() == null)
            return response;
       
        if (requiredFieldList.getFields()==null)
        {
            return response;
        }
       
        this.requiredFieldList = requiredFieldList;
        if(mLoadFunc instanceof LoadPushDown) {
            response = ((LoadPushDown)mLoadFunc).pushProjection(requiredFieldList);
        } else {
            // loadfunc does not support pushing projections
            response = new RequiredFieldResponse(false);
        }
        if (!response.getRequiredFieldResponse())
            return response;
       
        // Change LOLoad schema to reflect this pruning
        TreeSet<Integer> prunedIndexSet = new TreeSet<Integer>();
        for (int i=0;i<mSchema.size();i++)
            prunedIndexSet.add(i);

        for (int i=0;i<requiredFieldList.getFields().size();i++)
        {
            RequiredField requiredField = requiredFieldList.getFields().get(i);
            if (requiredField.getIndex()>=0)
                prunedIndexSet.remove(requiredField.getIndex());
            else
            {
               
                try {
                    int index = mSchema.getPosition(requiredField.getAlias());
                    if (index>0)
                        prunedIndexSet.remove(index);
                } catch (FrontendException e) {
                    return new RequiredFieldResponse(false);
                }
               
            }
        }
       
View Full Code Here
2000
2001
2002
2003
2004
2005
2006
            try {
                UDFContext.getUDFContext().getUDFProperties(this.getClass()).setProperty(signature, ObjectSerializer.serialize(aliases));
            } catch (IOException e) {
                throw new FrontendException(e);
            }
            return new RequiredFieldResponse(true);
        }
View Full Code Here
2000
2001
2002
2003
2004
2005
2006
            try {
                UDFContext.getUDFContext().getUDFProperties(this.getClass()).setProperty(signature, ObjectSerializer.serialize(aliases));
            } catch (IOException e) {
                throw new FrontendException(e);
            }
            return new RequiredFieldResponse(true);
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.LoadPushDown.RequiredFieldResponse

  • org.apache.pig.impl.logicalLayer.LOLoad
  • org.apache.pig.impl.logicalLayer.optimizer.PruneColumns
  • org.apache.pig.test.TestPruneColumn$PruneColumnEvalFunc
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.