Package org.apache.airavata.gfac.external

Examples of org.apache.airavata.gfac.external.GridFtp


        }


        ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
        ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
        GridFtp ftp = new GridFtp();

        try {
            GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentails();

            if (gridFTPEndpointArray == null || gridFTPEndpointArray.length == 0) {
              gridFTPEndpointArray = new String[]{hostType.getHostAddress()};
            }
            boolean success = false;
            GFacHandlerException pe = null;// = new ProviderException("");
            for (String endpoint : gridFTPEndpointArray) {
                try {

                    URI tmpdirURI = GFacUtils.createGsiftpURI(endpoint, app.getScratchWorkingDirectory());
                    URI workingDirURI = GFacUtils.createGsiftpURI(endpoint, app.getStaticWorkingDirectory());
                    URI inputURI = GFacUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
                    URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());

                    log.info("Host FTP = " + gridFTPEndpointArray[0]);
                    log.info("temp directory = " + tmpdirURI);
                    log.info("Working directory = " + workingDirURI);
                    log.info("Input directory = " + inputURI);
                    log.info("Output directory = " + outputURI);

                    ftp.makeDir(tmpdirURI, gssCred);
                    ftp.makeDir(workingDirURI, gssCred);
                    ftp.makeDir(inputURI, gssCred);
                    ftp.makeDir(outputURI, gssCred);

                    success = true;
                    break;
                } catch (URISyntaxException e) {
                    pe = new GFacHandlerException("URI is malformatted:" + e.getMessage(), e, jobExecutionContext);
View Full Code Here


          //TODO
        }


        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
        GridFtp ftp = new GridFtp();
        URI destURI = null;
        GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentails();

        for (String endpoint : gridFTPEndpointArray) {
            URI inputURI = GFacUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
            String fileName = new File(gridftpURL.getPath()).getName();
            String s = inputURI.getPath() + File.separator + fileName;
            //if user give a url just to refer an endpoint, not a web resource we are not doing any transfer
            if (fileName != null && !"".equals(fileName)) {
                destURI = GFacUtils.createGsiftpURI(endpoint, s);
                if (paramValue.startsWith("gsiftp")) {
                  // no need to do if it is unicore, as unicore will download this on user's behalf to the job space dir
                  if(isInputNonLocal) ftp.uploadFile(gridftpURL, destURI, gssCred);
                  else return paramValue;
                } else if (paramValue.startsWith("file")) {
                    String localFile = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
                    FileInputStream fis = null;
                    try {
                      fis = new FileInputStream(localFile);
                      ftp.uploadFile(destURI, gssCred, fis);
                    }finally {
                      fis.close();
                    }
                } else if (paramValue.startsWith("http")) {
                  // no need to do if it is unicore
                  if(isInputNonLocal) {
                    InputStream is = null;
                    try {
                      is = gridftpURL.toURL().openStream();
                      ftp.uploadFile(destURI, gssCred, (is));
                    }finally {
                      is.close();
                    }
                  } else {
                    // don't return destUri
View Full Code Here

       }
       else {
          //TODO
       }

       GridFtp ftp = new GridFtp();
       File localStdErrFile = null;
       Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
       try {
          GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentails();
          String[] hostgridFTP = gridFTPEndpointArray;
            if (hostgridFTP == null || hostgridFTP.length == 0) {
                hostgridFTP = new String[]{hostName};
            }
            for (String endpoint : gridFTPEndpointArray) {
                try {
                    /*
                     *  Read Stdout and Stderror
                     */
                    URI stdoutURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardOutput());
                    URI stderrURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardError());

                    log.info("STDOUT:" + stdoutURI.toString());
                    log.info("STDERR:" + stderrURI.toString());

                    File logDir = new File("./service_logs");
                    if (!logDir.exists()) {
                        logDir.mkdir();
                    }

                    String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext
                            .getServiceName());
                    File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout");
                    localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");




                    // Shahbaz Comment: in principle when job is failed the execution chain must be stopped
                    // there


                    String stdout = null;
                    String stderr = null;

                    // TODO: what if job is failed
                    // and this handler is not able to find std* files?
                    try {
                     stdout = ftp.readRemoteFile(stdoutURI, gssCred, localStdOutFile);
                     stderr = ftp.readRemoteFile(stderrURI, gssCred, localStdErrFile);
                     //TODO: do we also need to set them as output parameters for another job
                    }
                    catch(ToolsException e) {
                        log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished:  "+e.getMessage());
                    }


                    Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
                    Set<String> keys = output.keySet();
                    for (String paramName : keys) {
                        ActualParameter actualParameter = (ActualParameter) output.get(paramName);
                        if ("URIArray".equals(actualParameter.getType().getType().toString())) {
                            URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
                            List<String> outputList = ftp.listDir(outputURI, gssCred);
                            String[] valueList = outputList.toArray(new String[outputList.size()]);
                            ((URIArrayType) actualParameter.getType()).setValueArray(valueList);
                            // why to instantiate new instance?
//                            stringMap = new HashMap<String, ActualParameter>();
                            stringMap.put(paramName, actualParameter);
View Full Code Here

        Map<String, Object> parameters = output.getParameters();
        for (String paramName : parameters.keySet()) {
            ActualParameter actualParameter = (ActualParameter) parameters
                    .get(paramName);

            GridFtp ftp = new GridFtp();
            GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentails();
            try {
                if ("URI".equals(actualParameter.getType().getType().toString())) {
                    for (String endpoint : gridFTPEndpointArray) {
                        ((URIParameterType) actualParameter.getType()).setValue(doStaging(outputFileStagingPath,
View Full Code Here

          //TODO
        }


        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
        GridFtp ftp = new GridFtp();
        URI destURI = null;
        GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();

        for (String endpoint : gridFTPEndpointArray) {
            URI inputURI = GFacUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
            String fileName = new File(gridftpURL.getPath()).getName();
            String destLocalPath = inputURI.getPath() + File.separator + fileName;
            //if user give a url just to refer an endpoint, not a web resource we are not doing any transfer
            if (fileName != null && !"".equals(fileName)) {
                destURI = GFacUtils.createGsiftpURI(endpoint, destLocalPath);
                if (paramValue.startsWith("gsiftp")) {
                  // no need to do if it is unicore, as unicore will download this on user's behalf to the job space dir
                  if(isInputNonLocal) ftp.uploadFile(gridftpURL, destURI, gssCred);
                  else return paramValue;
                } else if (paramValue.startsWith("file")) {
                    String localFile = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
                    FileInputStream fis = null;
                    try {
                      fis = new FileInputStream(localFile);
                      ftp.uploadFile(destURI, gssCred, fis);
                    } catch (IOException e) {
                        throw new GFacException("Unable to create file : " + localFile ,e);
                    } finally {
                        if (fis != null) {
                            fis.close();
                        }
                    }
                } else if (paramValue.startsWith("http")) {
                  // no need to do if it is unicore
                  if(isInputNonLocal) {
                    InputStream is = null;
                    try {
                      is = gridftpURL.toURL().openStream();
                      ftp.uploadFile(destURI, gssCred, (is));
                    }finally {
                      is.close();
                    }
                  } else {
                    // don't return destUri
View Full Code Here

       }
       else {
          //TODO
       }

       GridFtp ftp = new GridFtp();
       File localStdErrFile = null;
       Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
       try {
          GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
          String[] hostgridFTP = gridFTPEndpointArray;
            if (hostgridFTP == null || hostgridFTP.length == 0) {
                hostgridFTP = new String[]{hostName};
            }
            for (String endpoint : gridFTPEndpointArray) {
                try {
                    /*
                     *  Read Stdout and Stderror
                     */
                    URI stdoutURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardOutput());
                    URI stderrURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardError());

                    log.info("STDOUT:" + stdoutURI.toString());
                    log.info("STDERR:" + stderrURI.toString());

                    File logDir = new File("./service_logs");
                    if (!logDir.exists()) {
                        logDir.mkdir();
                    }

                    String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext
                            .getServiceName());
                    File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout");
                    localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");


                    String stdout = null;
                    String stderr = null;

                    // TODO: what if job is failed
                    // and this handler is not able to find std* files?
                    try {
                     stdout = ftp.readRemoteFile(stdoutURI, gssCred, localStdOutFile);
                     stderr = ftp.readRemoteFile(stderrURI, gssCred, localStdErrFile);
                     //TODO: do we also need to set them as output parameters for another job
                     ApplicationDescription application = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
                     ApplicationDeploymentDescriptionType appDesc = application.getType();
                     appDesc.setStandardOutput(stdout);
                     appDesc.setStandardError(stderr);
                     jobExecutionContext.getApplicationContext().setApplicationDeploymentDescription(application);
                    }
                    catch(ToolsException e) {
                        log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished:  "+e.getMessage());
                    }


                    Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
                    Set<String> keys = output.keySet();
                    for (String paramName : keys) {
                        ActualParameter actualParameter = (ActualParameter) output.get(paramName);
                        if ("URIArray".equals(actualParameter.getType().getType().toString())) {
                            URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
                            List<String> outputList = ftp.listDir(outputURI, gssCred);
                            String[] valueList = outputList.toArray(new String[outputList.size()]);
                            ((URIArrayType) actualParameter.getType()).setValueArray(valueList);
                            // why to instantiate new instance?
//                            stringMap = new HashMap<String, ActualParameter>();
                            stringMap.put(paramName, actualParameter);
                        }else if ("StringArray".equals(actualParameter.getType().getType().toString())) {
                            String[] valueList = OutputUtils.parseStdoutArray(stdout, paramName);
                            ((StringArrayType) actualParameter.getType()).setValueArray(valueList);
//                            stringMap = new HashMap<String, ActualParameter>();
                            stringMap.put(paramName, actualParameter);
                        } else if ("URI".equals(actualParameter.getType().getType().toString())) {
                            URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
                              List<String> outputList = ftp.listDir(outputURI, gssCred);
              if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
                stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
              } else {
                String valueList = outputList.get(0);
                ((URIParameterType) actualParameter.getType()).setValue(valueList);
View Full Code Here

        Map<String, Object> parameters = output.getParameters();
        for (String paramName : parameters.keySet()) {
            ActualParameter actualParameter = (ActualParameter) parameters
                    .get(paramName);

            GridFtp ftp = new GridFtp();
            GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
            try {
                if ("URI".equals(actualParameter.getType().getType().toString())) {
                    for (String endpoint : gridFTPEndpointArray) {
                        ((URIParameterType) actualParameter.getType()).setValue(doStaging(outputFileStagingPath,
View Full Code Here

        }


        ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
        ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
        GridFtp ftp = new GridFtp();

        try {




            GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
                    getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();

            if (gridFTPEndpointArray == null || gridFTPEndpointArray.length == 0) {
              gridFTPEndpointArray = new String[]{hostType.getHostAddress()};
            }
            boolean success = false;
            GFacHandlerException pe = null;// = new ProviderException("");
            for (String endpoint : gridFTPEndpointArray) {
                try {

                    URI tmpdirURI = GFacUtils.createGsiftpURI(endpoint, app.getScratchWorkingDirectory());
                    URI workingDirURI = GFacUtils.createGsiftpURI(endpoint, app.getStaticWorkingDirectory());
                    URI inputURI = GFacUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
                    URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());

                    log.info("Host FTP = " + gridFTPEndpointArray[0]);
                    log.info("temp directory = " + tmpdirURI);
                    log.info("Working directory = " + workingDirURI);
                    log.info("Input directory = " + inputURI);
                    log.info("Output directory = " + outputURI);

                    ftp.makeDir(tmpdirURI, gssCred);
                    ftp.makeDir(workingDirURI, gssCred);
                    ftp.makeDir(inputURI, gssCred);
                    ftp.makeDir(outputURI, gssCred);

                    success = true;
                    break;
                } catch (URISyntaxException e) {
                    pe = new GFacHandlerException("URI is malformatted:" + e.getMessage(), e, jobExecutionContext);
View Full Code Here

          //TODO
        }


        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
        GridFtp ftp = new GridFtp();
        URI destURI = null;
        GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();

        for (String endpoint : gridFTPEndpointArray) {
            URI inputURI = GFacUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
            String fileName = new File(gridftpURL.getPath()).getName();
            fileName = ftp.gridFTPFileExist(inputURI, fileName,gssCred);

            String destLocalPath = inputURI.getPath() + File.separator + fileName;
            //if user give a url just to refer an endpoint, not a web resource we are not doing any transfer
            if (fileName != null && !"".equals(fileName)) {
                destURI = GFacUtils.createGsiftpURI(endpoint, destLocalPath);
                if (paramValue.startsWith("gsiftp")) {
                  // no need to do if it is unicore, as unicore will download this on user's behalf to the job space dir
                  if(isInputNonLocal) ftp.uploadFile(gridftpURL, destURI, gssCred);
                  else return paramValue;
                } else if (paramValue.startsWith("file")) {
                    String localFile = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
                    FileInputStream fis = null;
                    try {
                      fis = new FileInputStream(localFile);
                      ftp.uploadFile(destURI, gssCred, fis);
                    } catch (IOException e) {
                        throw new GFacException("Unable to create file : " + localFile ,e);
                    } finally {
                        if (fis != null) {
                            fis.close();
                        }
                    }
                } else if (paramValue.startsWith("http")) {
                  // no need to do if it is unicore
                  if(isInputNonLocal) {
                    InputStream is = null;
                    try {
                      is = gridftpURL.toURL().openStream();
                      ftp.uploadFile(destURI, gssCred, (is));
                    }finally {
                      is.close();
                    }
                  } else {
                    // don't return destUri
View Full Code Here

       }
       else {
          //TODO
       }

       GridFtp ftp = new GridFtp();
       File localStdErrFile = null;
       Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
       try {
          GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
          String[] hostgridFTP = gridFTPEndpointArray;
            if (hostgridFTP == null || hostgridFTP.length == 0) {
                hostgridFTP = new String[]{hostName};
            }
            for (String endpoint : gridFTPEndpointArray) {
                try {
                    /*
                     *  Read Stdout and Stderror
                     */
                    URI stdoutURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardOutput());
                    URI stderrURI = GFacUtils.createGsiftpURI(endpoint, app.getStandardError());

                    log.info("STDOUT:" + stdoutURI.toString());
                    log.info("STDERR:" + stderrURI.toString());

                    File logDir = new File("./service_logs");
                    if (!logDir.exists()) {
                        logDir.mkdir();
                    }

                    String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext
                            .getServiceName());
                    File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout");
                    localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");


                    String stdout = null;
                    String stderr = null;

                    // TODO: what if job is failed
                    // and this handler is not able to find std* files?
                    try {
                     stdout = ftp.readRemoteFile(stdoutURI, gssCred, localStdOutFile);
                     stderr = ftp.readRemoteFile(stderrURI, gssCred, localStdErrFile);
                     //TODO: do we also need to set them as output parameters for another job
                     ApplicationDescription application = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
                     ApplicationDeploymentDescriptionType appDesc = application.getType();
                     appDesc.setStandardOutput(stdout);
                     appDesc.setStandardError(stderr);
                     jobExecutionContext.getApplicationContext().setApplicationDeploymentDescription(application);
                    }
                    catch(ToolsException e) {
                        log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished:  "+e.getMessage());
                    }


                    Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
                    Set<String> keys = output.keySet();
                    for (String paramName : keys) {
                        ActualParameter actualParameter = (ActualParameter) output.get(paramName);
                        if ("URIArray".equals(actualParameter.getType().getType().toString())) {
                            URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
                            List<String> outputList = ftp.listDir(outputURI, gssCred);
                            String[] valueList = outputList.toArray(new String[outputList.size()]);
                            ((URIArrayType) actualParameter.getType()).setValueArray(valueList);
                            // why to instantiate new instance?
//                            stringMap = new HashMap<String, ActualParameter>();
                            stringMap.put(paramName, actualParameter);
                        }else if ("StringArray".equals(actualParameter.getType().getType().toString())) {
                            String[] valueList = OutputUtils.parseStdoutArray(stdout, paramName);
                            ((StringArrayType) actualParameter.getType()).setValueArray(valueList);
//                            stringMap = new HashMap<String, ActualParameter>();
                            stringMap.put(paramName, actualParameter);
                        } else if ("URI".equals(actualParameter.getType().getType().toString())) {
                            URI outputURI = GFacUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
                              List<String> outputList = ftp.listDir(outputURI, gssCred);
              if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
                stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
              } else {
                String valueList = outputList.get(0);
                ((URIParameterType) actualParameter.getType()).setValue(valueList);
View Full Code Here

TOP

Related Classes of org.apache.airavata.gfac.external.GridFtp

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.