Package org.apache.airavata.gsi.ssh.api

Examples of org.apache.airavata.gsi.ssh.api.Cluster


            } else {
                authenticationInfo = new DefaultPublicKeyFileAuthentication(sshPublicKey, sshPrivateKey, sshPrivateKeyPass);
            }
            ServerInfo serverInfo = new ServerInfo(sshUserName, registeredHost.getType().getHostAddress());

            Cluster pbsCluster = null;
            try {
                String installedParentPath = "/";
                if (((SSHHostType) registeredHost.getType()).getHpcResource()) {
                    installedParentPath = ((HpcApplicationDeploymentType)
                            jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
View Full Code Here


            authenticationInfo = new DefaultPublicKeyFileAuthentication(this.publicKeyPath, this.privateKeyPath,
                    this.passPhrase);
        }
            // Server info
            ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
            Cluster pbsCluster = null;
            // here doesn't matter what the job manager is because we are only doing some file handling
            // not really dealing with monitoring or job submission, so we pa
            pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
            String parentPath = inputPath + File.separator + jobExecutionContext.getExperimentID() + File.separator + jobExecutionContext.getTaskData().getTaskID();
            (new File(parentPath)).mkdirs();
View Full Code Here

        super.invoke(jobExecutionContext);
        makeDirectory(jobExecutionContext);
  }
  private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
                try {
        Cluster cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
        if (cluster == null) {
            throw new GFacHandlerException("Security context is not set properly");
        } else {
            log.info("Successfully retrieved the Security Context");
        }
        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();

            String workingDirectory = app.getScratchWorkingDirectory();
            cluster.makeDirectory(workingDirectory);
            cluster.makeDirectory(app.getScratchWorkingDirectory());
            cluster.makeDirectory(app.getInputDataDirectory());
            cluster.makeDirectory(app.getOutputDataDirectory());
            DataTransferDetails detail = new DataTransferDetails();
            TransferStatus status = new TransferStatus();
            status.setTransferState(TransferState.DIRECTORY_SETUP);
            detail.setTransferStatus(status);
            detail.setTransferDescription("Working directory = " + workingDirectory);
View Full Code Here

        // Server info
        ServerInfo serverInfo = new ServerInfo("ogce", "login1.stampede.tacc.utexas.edu",2222);


        Cluster pbsCluster = new
                PBSCluster(serverInfo, authenticationInfo, org.apache.airavata.gsi.ssh.util.CommonUtils.getPBSJobManager("/usr/bin/"));


        // Execute command
        System.out.println("Target PBS file path: " + workingDirectory);
        // constructing the job object
        JobDescriptor jobDescriptor = new JobDescriptor();
        jobDescriptor.setWorkingDirectory(workingDirectory);
        jobDescriptor.setShellName("/bin/bash");
        jobDescriptor.setJobName("GSI_SSH_SLEEP_JOB");
        jobDescriptor.setExecutablePath("/bin/echo");
        jobDescriptor.setAllEnvExport(true);
        jobDescriptor.setMailOptions("n");
        jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
        jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
        jobDescriptor.setNodes(1);
        jobDescriptor.setProcessesPerNode(1);
        jobDescriptor.setQueueName("normal");
        jobDescriptor.setMaxWallTime("60");
        jobDescriptor.setAcountString("TG-STA110014S");
        List<String> inputs = new ArrayList<String>();
        jobDescriptor.setOwner("ogce");
        inputs.add("Hello World");
        jobDescriptor.setInputValues(inputs);
        //finished construction of job object
        System.out.println(jobDescriptor.toXML());
        String jobID = pbsCluster.submitBatchJob(jobDescriptor);
        System.out.println(jobID);
        try {
            pushQueue.add(new MonitorID(hostDescription, jobID,null,null,null, "ogce"));
        } catch (Exception e) {
            e.printStackTrace();
View Full Code Here

        TransferStatus status = new TransferStatus();

        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
                .getApplicationDeploymentDescription().getType();
        try {
            Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
            if (cluster == null) {
                throw new GFacProviderException("Security context is not set properly");
            } else {
                log.info("Successfully retrieved the Security Context");
            }

            // Get the Stdouts and StdErrs
            String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName());

            TaskDetails taskData = jobExecutionContext.getTaskData();
            String outputDataDir = null;
            File localStdOutFile;
            File localStdErrFile;

            if (taskData.getAdvancedOutputDataHandling() != null) {
                outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
            }
            if (outputDataDir == null) {
                outputDataDir = File.separator + "tmp";
            }
            outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID();
            (new File(outputDataDir)).mkdirs();


            localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout");
            localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr");
//            cluster.makeDirectory(outputDataDir);
            cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
            Thread.sleep(1000);
            cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
            Thread.sleep(1000);

            String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
            String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
            status.setTransferState(TransferState.COMPLETE);
            detail.setTransferStatus(status);
            detail.setTransferDescription("STDOUT:" + stdOutStr);
            registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());

            status.setTransferState(TransferState.COMPLETE);
            detail.setTransferStatus(status);
            detail.setTransferDescription("STDERR:" + stdErrStr);
            registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());


            List<DataObjectType> outputArray = new ArrayList<DataObjectType>();
            Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
            Set<String> keys = output.keySet();
            for (String paramName : keys) {
                ActualParameter actualParameter = (ActualParameter) output.get(paramName);
                if ("URI".equals(actualParameter.getType().getType().toString())) {

                    List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
                    if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
                        OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr,outputArray);
                        break;
                    } else {
                        String valueList = outputList.get(0);
                        cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
                        jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
                        DataObjectType dataObjectType = new DataObjectType();
                        dataObjectType.setValue(valueList);
                        dataObjectType.setKey(paramName);
                        dataObjectType.setType(DataType.URI);
View Full Code Here

        }
        jobExecutionContext.setInMessageContext(inputNew);
    }

    private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
        Cluster cluster = null;
        if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
            cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
        } else {
            cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
        }
        if (cluster == null) {
            throw new GFacException("Security context is not set properly");
        } else {
            log.info("Successfully retrieved the Security Context");
        }
        ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
        int i = paramValue.lastIndexOf(File.separator);
        String substring = paramValue.substring(i + 1);
        try {
            String targetFile = app.getInputDataDirectory() + File.separator + substring;
            if (paramValue.startsWith("file")) {
                paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
            }
            cluster.scpTo(targetFile, paramValue);
            return targetFile;
        } catch (SSHApiException e) {
            throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
        }
    }
View Full Code Here

                    requestData.getMyProxyPort(), requestData.getMyProxyLifeTime(), System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY));
            GsisshHostType gsisshHostType = (GsisshHostType) registeredHost.getType();
            ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress(),
                    gsisshHostType.getPort());

            Cluster pbsCluster = null;
            try {
                JobManagerConfiguration jConfig = null;
                String installedParentPath = ((HpcApplicationDeploymentType)
                        jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
                String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager();
View Full Code Here

                HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
                        getApplicationDeploymentDescription().getType();
                JobDetails jobDetails = new JobDetails();
                String taskID = jobExecutionContext.getTaskData().getTaskID();
                try {
                    Cluster cluster = null;
                    if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) != null) {
                        cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
                    }
                    if (cluster == null) {
                        throw new GFacProviderException("Security context is not set properly");
                    } else {
                        log.info("Successfully retrieved the Security Context");
                    }
                    // This installed path is a mandetory field, because this could change based on the computing resource
                    JobDescriptor jobDescriptor = GFACSSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);

                    log.info(jobDescriptor.toXML());

                    jobDetails.setJobDescription(jobDescriptor.toXML());

                    String jobID = cluster.submitBatchJob(jobDescriptor);
                    jobExecutionContext.setJobDetails(jobDetails);
                    if (jobID == null) {
                        jobDetails.setJobID("none");
                        GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
                    } else {
View Full Code Here

                = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
                7512, 17280000, certificateLocation);

        // Server info
        ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
        Cluster pbsCluster = null;
        try {
            pbsCluster = new PBSCluster(serverInfo, authenticationInfo, app.getInstalledParentPath());
        } catch (SSHApiException e) {
            e.printStackTrace()//To change body of catch statement use File | Settings | File Templates.
        }
View Full Code Here

        HostDescriptionType host = jobExecutionContext.getApplicationContext().
                getHostDescription().getType();
        HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
                getApplicationDeploymentDescription().getType();
        try {
            Cluster cluster = null;
            if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
                cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
            } else {
                cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
            }
            if (cluster == null) {
                throw new GFacProviderException("Security context is not set properly");
            } else {
                log.info("Successfully retrieved the Security Context");
            }
            // This installed path is a mandetory field, because this could change based on the computing resource
            JobDescriptor jobDescriptor = new JobDescriptor();
            jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
            jobDescriptor.setShellName("/bin/bash");
            Random random = new Random();
            int i = random.nextInt();
            jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
            jobDescriptor.setExecutablePath(app.getExecutableLocation());
            jobDescriptor.setAllEnvExport(true);
            jobDescriptor.setMailOptions("n");
            jobDescriptor.setStandardOutFile(app.getStandardOutput());
            jobDescriptor.setStandardErrorFile(app.getStandardError());
            jobDescriptor.setNodes(app.getNodeCount());
            jobDescriptor.setProcessesPerNode(app.getProcessorsPerNode());
            jobDescriptor.setMaxWallTime(String.valueOf(app.getMaxWallTime()));
            jobDescriptor.setJobSubmitter(app.getJobSubmitterCommand());
            if (app.getProjectAccount().getProjectAccountNumber() != null) {
                jobDescriptor.setAcountString(app.getProjectAccount().getProjectAccountNumber());
            }
            if (app.getQueue().getQueueName() != null) {
                jobDescriptor.setQueueName(app.getQueue().getQueueName());
            }
            jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
            List<String> inputValues = new ArrayList<String>();
            MessageContext input = jobExecutionContext.getInMessageContext();
            Map<String, Object> inputs = input.getParameters();
            Set<String> keys = inputs.keySet();
            for (String paramName : keys) {
                ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
                if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
                        || "FileArray".equals(actualParameter.getType().getType().toString())) {
                    String[] values = null;
                    if (actualParameter.getType() instanceof URIArrayType) {
                        values = ((URIArrayType) actualParameter.getType()).getValueArray();
                    } else if (actualParameter.getType() instanceof StringArrayType) {
                        values = ((StringArrayType) actualParameter.getType()).getValueArray();
                    } else if (actualParameter.getType() instanceof FileArrayType) {
                        values = ((FileArrayType) actualParameter.getType()).getValueArray();
                    }
                    String value = StringUtil.createDelimiteredString(values, " ");
                    inputValues.add(value);
                } else {
                    String paramValue = MappingFactory.toString(actualParameter);
                    inputValues.add(paramValue);
                }
            }
            jobDescriptor.setInputValues(inputValues);

            log.info(jobDescriptor.toXML());
            final String jobID = cluster.submitBatchJob(jobDescriptor);
            log.info("Job Submitted successfully and returned Job ID: " + jobID);
            jobExecutionContext.getNotifier().publish(new JobIDEvent(jobID));

            final JobSubmissionListener listener = new GSISSHJobSubmissionListener(jobExecutionContext);
            final Cluster finalCluster = cluster;
//            try {
//            // Wait 5 seconds to start the first poll, this is hard coded, user doesn't have
//            // to configure this.
//            Thread.sleep(5000);
//        } catch (InterruptedException e) {
//            log.error("Error during job status monitoring");
//            throw new SSHApiException("Error during job status monitoring", e);
//        }
//        // Get the job status first
            try {
//
                Thread t = new Thread() {
                    @Override
                    public void run() {
                        try {
                            JobStatus jobStatus = finalCluster.getJobStatus(jobID);
                            listener.statusChanged(jobStatus);
                            while (true) {
                                while (!jobStatus.equals(JobStatus.C)) {
                                    if (!jobStatus.equals(listener.getJobStatus().toString())) {
                                        listener.setJobStatus(jobStatus);
                                        listener.statusChanged(jobStatus);
                                    }
                                    Thread.sleep(60000);

                                    jobStatus = finalCluster.getJobStatus(jobID);
                                }
                                //Set the job status to Complete
                                listener.setJobStatus(JobStatus.C);
                                listener.statusChanged(jobStatus);
                                break;
View Full Code Here

TOP

Related Classes of org.apache.airavata.gsi.ssh.api.Cluster

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.