Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.ElementDescriptor

326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
    }
   
    static public InputStream open(String fileSpec, PigContext pigContext) throws IOException {
        fileSpec = checkDefaultPrefix(pigContext.getExecType(), fileSpec);
        if (!fileSpec.startsWith(LOCAL_PREFIX)) {
            ElementDescriptor elem = pigContext.getDfs().asElement(fullPath(fileSpec, pigContext));
            return openDFSFile(elem);
        }
        else {
            fileSpec = fileSpec.substring(LOCAL_PREFIX.length());
            //buffering because we only want buffered streams to be passed to load functions.
            /*return new BufferedInputStream(new FileInputStream(fileSpec));*/
            ElementDescriptor elem = pigContext.getLfs().asElement(fullPath(fileSpec, pigContext));
            return openLFSFile(elem);
        }
    }
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
     */
    static public SeekableInputStream open(String fileSpec, long offset, PigContext pigContext) throws IOException {
       
        fileSpec = checkDefaultPrefix(pigContext.getExecType(), fileSpec);
       
        ElementDescriptor elem;
        if (!fileSpec.startsWith(LOCAL_PREFIX))
            elem = pigContext.getDfs().asElement(fullPath(fileSpec, pigContext));
               
        else{
            fileSpec = fileSpec.substring(LOCAL_PREFIX.length());
            elem = pigContext.getLfs().asElement(fullPath(fileSpec, pigContext));           
        }
       
        if (elem.exists() && (!elem.getDataStorage().isContainer(elem.toString()))) {
            try {
                if (elem.systemElement())
                    throw new IOException ("Attempt is made to open system file " + elem.toString());
               
                SeekableInputStream sis = elem.sopen();
                sis.seek(offset, FLAGS.SEEK_SET);
                return sis;
            }
            catch (DataStorageException e) {
                throw new IOException("Failed to determine if elem=" + elem + " is container", e);
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
397
    }

    static public OutputStream create(String fileSpec, boolean append, PigContext pigContext) throws IOException {
        fileSpec = checkDefaultPrefix(pigContext.getExecType(), fileSpec);
        if (!fileSpec.startsWith(LOCAL_PREFIX)) {
            ElementDescriptor elem = pigContext.getDfs().asElement(fileSpec);
            return elem.create();
        }
        else {
            fileSpec = fileSpec.substring(LOCAL_PREFIX.length());

            // TODO probably this should be replaced with the local file system
View Full Code Here
407
408
409
410
411
412
413
414
415
416
417
418
    }
   
    static public boolean delete(String fileSpec, PigContext pigContext) throws IOException{
        fileSpec = checkDefaultPrefix(pigContext.getExecType(), fileSpec);
        if (!fileSpec.startsWith(LOCAL_PREFIX)) {
            ElementDescriptor elem = pigContext.getDfs().asElement(fileSpec);
            elem.delete();
            return true;
        }
        else {
            fileSpec = fileSpec.substring(LOCAL_PREFIX.length());
            boolean ret = true;
View Full Code Here
491
492
493
494
495
496
497
498
499
500
501
502
    }

    public static void deleteTempFiles() {
        while (!toDelete().empty()) {
            try {
                ElementDescriptor elem = toDelete().pop();
                elem.delete();
            }
            catch (IOException e) {
                log.error(e);
            }
        }
View Full Code Here
514
515
516
517
518
519
520
521
522
523
            relative = relativeRoot(pigContext);
        }
        if (!relativeRoot(pigContext).exists()) {
            relativeRoot(pigContext).create();
        }
        ElementDescriptor elem=
            pigContext.getDfs().asElement(relative.toString(), "tmp" + r.nextInt());
        toDelete().push(elem);
        return elem;
    }
View Full Code Here
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
        toDelete().push(elem);
        return elem;
    }

    public static Path getTemporaryPath(PigContext pigContext) throws IOException {
        ElementDescriptor relative = relativeRoot(pigContext);
   
        if (!relativeRoot(pigContext).exists()) {
            relativeRoot(pigContext).create();
        }
        ElementDescriptor elem=
            pigContext.getDfs().asElement(relative.toString(), "tmp" + r.nextInt());
        toDelete().push(elem);
        return ((HPath)elem).getPath();
    }
View Full Code Here
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
    public static String hadoopify(String filename, PigContext pigContext) throws IOException {
        if (filename.startsWith(LOCAL_PREFIX)) {
            filename = filename.substring(LOCAL_PREFIX.length());
        }
       
        ElementDescriptor localElem =
            pigContext.getLfs().asElement(filename);
           
        if (!localElem.exists()) {
            throw new FileNotFoundException(filename);
        }
           
        ElementDescriptor distribElem =
            getTemporaryPath(null, pigContext);
   
        int suffixStart = filename.lastIndexOf('.');
        if (suffixStart != -1) {
            distribElem = pigContext.getDfs().asElement(distribElem.toString() +
                    filename.substring(suffixStart));
        }
           
        // TODO: currently the copy method in Data Storage does not allow to specify overwrite
        //       so the work around is to delete the dst file first, if it exists
        if (distribElem.exists()) {
            distribElem.delete();
        }
        localElem.copy(distribElem, null, false);
           
        return distribElem.toString();
    }
View Full Code Here
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
    }

    public static String fullPath(String filename, PigContext pigContext) throws IOException {
        try {
            if (filename.charAt(0) != '/') {
                ElementDescriptor currentDir = pigContext.getDfs().getActiveContainer();
                ElementDescriptor elem = pigContext.getDfs().asElement(currentDir.toString(),
                                                                                  filename);
               
                return elem.toString();
            }
            return filename;
        }
        catch (DataStorageException e) {
            return filename;
View Full Code Here
590
591
592
593
594
595
596
597
     * @deprecated Use {@link #fileExists(String, PigContext)} instead
     */
    @Deprecated
    public static boolean fileExists(String filename, DataStorage store)
            throws IOException {
        ElementDescriptor elem = store.asElement(filename);
        return elem.exists() || globMatchesFiles(elem, store);
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.datastorage.ElementDescriptor

  • com.packetloop.packetpig.storage.JsonMetadata
  • org.apache.pig.backend.executionengine.PigSlicer
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.MapReduceLauncher
  • org.apache.pig.backend.local.datastorage.LocalDir
  • org.apache.pig.builtin.JsonMetadata
  • org.apache.pig.impl.io.FileLocalizer
  • org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor
  • org.apache.pig.impl.PigContext
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.piggybank.storage.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.