Examples of edu.sdsc.grid.io.GeneralFile

edu.sdsc.grid.io.GeneralFile


        SRBFile file = new SRBFile(client, "/ngs/home");
        GeneralFile[] kids = file.listFiles(); 
        if(kids != null){
            for(int i=0; i<kids.length; i++){
                GeneralFile child = kids[i]; 
                String name = child.getName();
                boolean isDir = child.isDirectory();
                boolean isFile = child.isFile(); 
                
                
                System.out.println("name: "+name+" "+isDir+" "+isFile);
            }
        }

View Full Code Here

            } else {
                log.debug("children are present: " + kids.length);
                children = new TreeMap<String, SrbGeneralFileFileObject>();


                for (int i = 0; i < kids.length; i++) {
                    final GeneralFile child = kids[i];
                    if (child == null) {
                        continue;
                    }
                    String childname = child.getName();




                    if (childname != null && !".".equals(childname) && !"..".equals(childname) &&
                            !"./".equals(childname) && !"../".equals(childname)) {


                        // Resolves a name, relative to the "root" file name. 
                        // Refer to NameScope  for a description of how names are resolved.
                        //
                        // Parameters:
                        // root - the base filename
                        // name - The name to resolve.
                        // scope - The NameScope to use when resolving the name.


                        // update the child FileObjects GeneralFile. 
                        FileName fn = getFileSystem().getFileSystemManager().
                                resolveName(getName(), UriParser.encode(childname), NameScope.CHILD);
                        //log.debug("childname: [" + childname + "] fnStr: [" + fn.toString() + "] fnBas: [" + fn.getBaseName() + "]");


                        /** 
                         * VFS resolveName and Jargon GeneralFile.getName() bug. 
                         * Refer to results of following debug statement in explanation:
                         * log.debug("childname: [" + childname + "] fnStr: [" + fn.toString() + "] fnBas: [" + fn.getBaseName() + "]");
                         * 
                         * A) Output of SRB Sls:
                         * =====================
                         * [ngs0153@ngs ~]$ Sls /ngs/home 
                         *    C-/ngs/home/http://marc.theaimsgroup.com/?l=redhat-list&m=109219329520329&w=2.ngs
                         *    C-/home/ngs/david-meredith.ngs
                         *    ...
                         * Note, that the file listing above contains a collection whose name
                         * begins with a url "http;//marc.theaimsgroup.com.."
                         *                        
                         * 
                         * B) Name retireved by Jargon (child.getName()):
                         * ==============================================
                         *    childname: [?l=redhat-list&m=109219329520329&w=2.ngs]
                         *    
                         * Note, that the child name does not include the url 
                         * part of the file name ("http;//marc.theaimsgroup.com.."). 
                         * 
                         *                        
                         * D) FileName resolved by VFS (fn above)
                         * ========================================
                         *    fnStr:     [srb://srb1.ngs.rl.ac.uk/ngs/home?l=redhat-list&m=109219329520329&w=2.ngs]
                         *    fnBas:     [home]
                         *
                         * Note that the fn.toString() does not include the url part of the file name and also drops the 
                         * file separator betweeen the home dir and the file name ("/ngs/home?l=redhat..."). 
                         * As a result, the fn.getBaseName() returns 'home' which is wrong. 
                         */
                        // Due to above bug, need to check that childname == fn.getBaseName() 
                        // otherwise, will not be able to resolveFile correctly which causes ex. 
                        if (fn != null && childname != null && childname.equals(fn.getBaseName())) {
                            //&& !"srb://srb1.ngs.rl.ac.uk/ngs/home?l=redhat-list&m=109219329520329&w=2.ngs".equals(fn.toString())) {


                            SrbGeneralFileFileObject fo = (SrbGeneralFileFileObject) getFileSystem().resolveFile(fn);
                            // Could also pass string to resolveFile and this must be an absPath !
                            //SrbGeneralFileFileObject fo = (SrbGeneralFileFileObject) getFileSystem().resolveFile(getName().getPath() + getName().SEPARATOR + childname);


                            // update the child collection. 
                            if (fo != null) {
                                //fo.file = FileFactory.newFile(client, filePath + "/" + name);
                                fo.file = child;


                                // set this FileObjects FileType by injection 
                                if (child.isFile()) {
                                    // instantiate fo.attribs, fo.fileAttributes ?
                                    fo.injectType(FileType.FILE);
                                    children.put(childname, fo);


                                } else if (child.isDirectory()) {
                                    // instantiate fo.attribs, fo.fileAttributes ?
                                    fo.injectType(FileType.FOLDER);
                                    children.put(childname, fo);


                                }

View Full Code Here


            throw sqle;
        }


        // Where on the file system will this new bitstream go?
    GeneralFile file = getFile(bitstream);


        // Make the parent dirs if necessary
    GeneralFile parent = file.getParentFile();


        if (!parent.exists())
        {
            parent.mkdirs();
        }


        //Create the corresponding file and open it
        file.createNewFile();

View Full Code Here

      }
      throw sqle;
    }


    // get a reference to the file
    GeneralFile file = getFile(bitstream);


    // read through a DigestInputStream that will work out the MD5
    //
    // DSpace refers to checksum, writes it in METS, and uses it as an
    // AIP filename (!), but never seems to validate with it. Furthermore,
    // DSpace appears to hardcode the algorithm to MD5 in some places--see 
    // METSExport.java.
    //
    // To remain compatible with DSpace we calculate an MD5 checksum on 
    // LOCAL registered files. But for REMOTE (e.g. SRB) files we 
    // calculate an MD5 on just the fileNAME. The reasoning is that in the 
    // case of a remote file, calculating an MD5 on the file itself will
    // generate network traffic to read the file's bytes. In this case it 
    // would be better have a proxy process calculate MD5 and store it as 
    // an SRB metadata attribute so it can be retrieved simply from SRB.
    //
    // TODO set this up as a proxy server process so no net activity
    
    // FIXME this is a first class HACK! for the reasons described above
    if (file instanceof LocalFile) 
    {


      // get MD5 on the file for local file
      DigestInputStream dis = null;
      try 
      {
        dis = new DigestInputStream(FileFactory.newFileInputStream(file), 
            MessageDigest.getInstance("MD5"));
      } 
      catch (NoSuchAlgorithmException e) 
      {
        log.warn("Caught NoSuchAlgorithmException", e);
        throw new IOException("Invalid checksum algorithm");
      }
      catch (IOException e) 
      {
        log.error("File: " + file.getAbsolutePath() 
            + " to be registered cannot be opened - is it "
            + "really there?");
        throw e;
      }
      final int BUFFER_SIZE = 1024 * 4;
      final byte[] buffer = new byte[BUFFER_SIZE];
      while (true) 
      {
        final int count = dis.read(buffer, 0, BUFFER_SIZE);
        if (count == -1) 
        {
          break;
        }
      }
      bitstream.setColumn("checksum", Utils.toHex(dis.getMessageDigest()
          .digest()));
      dis.close();
    } 
    else if (file instanceof SRBFile)
    {
      if (!file.exists())
      {
        log.error("File: " + file.getAbsolutePath() 
            + " is not in SRB MCAT");
        throw new IOException("File is not in SRB MCAT");
      }


      // get MD5 on just the filename (!) for SRB file
      int iLastSlash = bitstreamPath.lastIndexOf('/');
      String sFilename = bitstreamPath.substring(iLastSlash + 1);
      MessageDigest md = null;
      try 
      {
        md = MessageDigest.getInstance("MD5");
      } 
      catch (NoSuchAlgorithmException e) 
      {
        log.error("Caught NoSuchAlgorithmException", e);
        throw new IOException("Invalid checksum algorithm");
      }
      bitstream.setColumn("checksum", 
          Utils.toHex(md.digest(sFilename.getBytes())));
    }
    else
    {
      throw new IOException("Unrecognized file type - "
          + "not local, not SRB");
    }


    bitstream.setColumn("checksum_algorithm", "MD5");
    bitstream.setColumn("size_bytes", file.length());
    bitstream.setColumn("deleted", false);
    DatabaseManager.update(context, bitstream);


    int bitstream_id = bitstream.getIntColumn("bitstream_id");
    if (log.isDebugEnabled()) 
    {
      log.debug("Stored bitstream " + bitstream_id + " in file "
          + file.getAbsolutePath());
    }
    return bitstream_id;
  }

View Full Code Here

    public static InputStream retrieve(Context context, int id)
            throws SQLException, IOException
    {
        TableRow bitstream = DatabaseManager.find(context, "bitstream", id);


    GeneralFile file = getFile(bitstream);


    return (file != null) ? FileFactory.newFileInputStream(file) : null;
    }

View Full Code Here

            for (Iterator iterator = storage.iterator(); iterator.hasNext();)
            {
                TableRow row = (TableRow) iterator.next();
                int bid = row.getIntColumn("bitstream_id");


        GeneralFile file = getFile(row);


                // Make sure entries which do not exist are removed
                if (file == null || !file.exists())
                {
                    log.debug("file is null");
                    if (deleteDbRecords)
                    {
                        log.debug("deleting record");
                        if (verbose) System.out.println(" - Deleting bitstream information (ID: " + bid + ")");
                        bitstreamInfoDAO.deleteBitstreamInfoWithHistory(bid);
                        if (verbose) System.out.println(" - Deleting bitstream record from database (ID: " + bid + ")");
                        DatabaseManager.delete(context, "Bitstream", bid);
                    }
                    continue;
                }


                // This is a small chance that this is a file which is
                // being stored -- get it next time.
                if (isRecent(file))
                {
                  log.debug("file is recent");
                    continue;
                }


                if (deleteDbRecords)
                {
                    log.debug("deleting db record");
                    if (verbose) System.out.println(" - Deleting bitstream information (ID: " + bid + ")");
                    bitstreamInfoDAO.deleteBitstreamInfoWithHistory(bid);
                    if (verbose) System.out.println(" - Deleting bitstream record from database (ID: " + bid + ")");
                    DatabaseManager.delete(context, "Bitstream", bid);
                }


        if (isRegisteredBitstream(row.getStringColumn("internal_id"))) {
            continue;      // do not delete registered bitstreams
        }


                boolean success = file.delete();


                String message = ("Deleted bitstream " + bid + " (file "
                            + file.getAbsolutePath() + ") with result "
                            + success);
                if (log.isDebugEnabled())
                {
                    log.debug(message);
                }

View Full Code Here

        if (file == null )
        {
            return;
        }
 
    GeneralFile tmp = file;


        for (int i = 0; i < directoryLevels; i++)
        {


      GeneralFile directory = tmp.getParentFile();
      GeneralFile[] files = directory.listFiles();


            // Only delete empty directories
            if (files.length != 0)
            {
                break;
            }


            directory.delete();
            tmp = directory;
        }
    }

View Full Code Here

        if (storeNumber == -1)
        {
            storeNumber = 0;
        }


    GeneralFile assetstore = assetStores[storeNumber];


    // turn the internal_id into a file path relative to the assetstore
    // directory
    String sInternalId = bitstream.getStringColumn("internal_id");


    // there are 4 cases:
    // -conventional bitstream, conventional storage
    // -conventional bitstream, srb storage
    // -registered bitstream, conventional storage
    // -registered bitstream, srb storage
    // conventional bitstream - dspace ingested, dspace random name/path
    // registered bitstream - registered to dspace, any name/path
    String sIntermediatePath = null;
    if (isRegisteredBitstream(sInternalId)) {
      sInternalId = sInternalId.substring(REGISTERED_FLAG.length());
      sIntermediatePath = "";
    } else {
      
      // Sanity Check: If the internal ID contains a
      // pathname separator, it's probably an attempt to
      // make a path traversal attack, so ignore the path
      // prefix.  The internal-ID is supposed to be just a
      // filename, so this will not affect normal operation.
      if (sInternalId.indexOf(File.separator) != -1)
        sInternalId = sInternalId.substring(sInternalId.lastIndexOf(File.separator)+1);
      
      sIntermediatePath = getIntermediatePath(sInternalId);
    }


    StringBuffer bufFilename = new StringBuffer();
    if (assetstore instanceof LocalFile) {
      bufFilename.append(assetstore.getCanonicalPath());
      bufFilename.append(File.separator);
      bufFilename.append(sIntermediatePath);
      bufFilename.append(sInternalId);
      if (log.isDebugEnabled()) {
        log.debug("Local filename for " + sInternalId + " is "

View Full Code Here


            throw sqle;
        }


        // Where on the file system will this new bitstream go?
    GeneralFile file = getFile(bitstream);


        // Make the parent dirs if necessary
    GeneralFile parent = file.getParentFile();


        if (!parent.exists())
        {
            parent.mkdirs();
        }


        //Create the corresponding file and open it
        file.createNewFile();

View Full Code Here

      }
      throw sqle;
    }


    // get a reference to the file
    GeneralFile file = getFile(bitstream);


    // read through a DigestInputStream that will work out the MD5
    //
    // DSpace refers to checksum, writes it in METS, and uses it as an
    // AIP filename (!), but never seems to validate with it. Furthermore,
    // DSpace appears to hardcode the algorithm to MD5 in some places--see 
    // METSExport.java.
    //
    // To remain compatible with DSpace we calculate an MD5 checksum on 
    // LOCAL registered files. But for REMOTE (e.g. SRB) files we 
    // calculate an MD5 on just the fileNAME. The reasoning is that in the 
    // case of a remote file, calculating an MD5 on the file itself will
    // generate network traffic to read the file's bytes. In this case it 
    // would be better have a proxy process calculate MD5 and store it as 
    // an SRB metadata attribute so it can be retrieved simply from SRB.
    //
    // TODO set this up as a proxy server process so no net activity
    
    // FIXME this is a first class HACK! for the reasons described above
    if (file instanceof LocalFile) 
    {


      // get MD5 on the file for local file
      DigestInputStream dis = null;
      try 
      {
        dis = new DigestInputStream(FileFactory.newFileInputStream(file), 
            MessageDigest.getInstance("MD5"));
      } 
      catch (NoSuchAlgorithmException e) 
      {
        log.warn("Caught NoSuchAlgorithmException", e);
        throw new IOException("Invalid checksum algorithm", e);
      }
      catch (IOException e) 
      {
        log.error("File: " + file.getAbsolutePath() 
            + " to be registered cannot be opened - is it "
            + "really there?");
        throw e;
      }
      final int BUFFER_SIZE = 1024 * 4;
      final byte[] buffer = new byte[BUFFER_SIZE];
      while (true) 
      {
        final int count = dis.read(buffer, 0, BUFFER_SIZE);
        if (count == -1) 
        {
          break;
        }
      }
      bitstream.setColumn("checksum", Utils.toHex(dis.getMessageDigest()
          .digest()));
      dis.close();
    } 
    else if (file instanceof SRBFile)
    {
      if (!file.exists())
      {
        log.error("File: " + file.getAbsolutePath() 
            + " is not in SRB MCAT");
        throw new IOException("File is not in SRB MCAT");
      }


      // get MD5 on just the filename (!) for SRB file
      int iLastSlash = bitstreamPath.lastIndexOf('/');
      String sFilename = bitstreamPath.substring(iLastSlash + 1);
      MessageDigest md = null;
      try 
      {
        md = MessageDigest.getInstance("MD5");
      } 
      catch (NoSuchAlgorithmException e) 
      {
        log.error("Caught NoSuchAlgorithmException", e);
        throw new IOException("Invalid checksum algorithm", e);
      }
      bitstream.setColumn("checksum", 
          Utils.toHex(md.digest(sFilename.getBytes())));
    }
    else
    {
      throw new IOException("Unrecognized file type - "
          + "not local, not SRB");
    }


    bitstream.setColumn("checksum_algorithm", "MD5");
    bitstream.setColumn("size_bytes", file.length());
    bitstream.setColumn("deleted", false);
    DatabaseManager.update(context, bitstream);


    int bitstreamId = bitstream.getIntColumn("bitstream_id");
    if (log.isDebugEnabled()) 
    {
      log.debug("Stored bitstream " + bitstreamId + " in file "
          + file.getAbsolutePath());
    }
    return bitstreamId;
  }

View Full Code Here

0 1

TOP

Related Classes of edu.sdsc.grid.io.GeneralFile

org.apache.commons.vfs.provider.srb.jargon.SrbGeneralFileFileObject

org.dspace.storage.bitstore.BitstreamStorageManager

uk.ac.dl.escience.junkdev.tests.JargonSrbTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.