Examples of org.apache.oozie.service.HadoopAccessorService

org.apache.oozie.service.HadoopAccessorService
The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group.
The default accessor used is the base accessor which just injects the UGI into the configuration instance used to create/obtain JobClient and ileSystem instances.
The HadoopAccess class to use can be configured in the oozie-site.xml using the oozie.service.HadoopAccessorService.accessor.class property.

        String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
        // Configuration confHadoop = CoordUtils.getHadoopConf(conf);
        try {
            URI uri = new URI(appPath);
            LOG.debug("user =" + user);
            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
            Configuration fsConf = has.createJobConf(uri.getAuthority());
            FileSystem fs = has.createFileSystem(user, uri, fsConf);
            Path appDefPath = null;


            // app path could be a directory
            Path path = new Path(uri.getPath());
            // check file exists for dataset include file, app xml already checked

View Full Code Here

    private static Class<? extends AuthHelper> KLASS = null;


    @SuppressWarnings("unchecked")
    public synchronized static AuthHelper get() {
        if (KLASS == null) {
            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
            if (has.getClass() ==  HadoopAccessorService.class) {
                KLASS = AuthHelper.class;
            }
            else {
                try {
                    KLASS = (Class<? extends AuthHelper>) Class

View Full Code Here

            if (e != null) {
                String strConf = XmlUtils.prettyPrint(e).toString();
                XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));


                XConfiguration launcherConf = new XConfiguration();
                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
                XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_JOB_TRACKER), getType());
                injectLauncherProperties(actionDefaultConf, launcherConf);
                injectLauncherProperties(inlineConf, launcherConf);
                injectLauncherUseUberMode(launcherConf);
                checkForDisallowedProps(launcherConf, "launcher configuration");
                XConfiguration.copy(launcherConf, conf);

View Full Code Here


    }


    private FileSystem getTargetFileSysyem() throws Exception {
        if (fs == null) {
            HadoopAccessorService has = getServices().get(HadoopAccessorService.class);
            URI uri = new Path(outPath).toUri();
            Configuration fsConf = has.createJobConf(uri.getAuthority());
            fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
        }
        return fs;


    }

View Full Code Here


    private static FileSystem getFileSystem(URI uri) throws HadoopAccessorException {
        WorkflowJob workflow = DagELFunctions.getWorkflow();
        String user = workflow.getUser();
        String group = workflow.getGroup();
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        JobConf conf = has.createJobConf(uri.getAuthority());
        return has.createFileSystem(user, uri, conf);
    }

View Full Code Here

    public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf)
            throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
        Namespace ns = element.getNamespace();
        Iterator<Element> it = element.getChildren("job-xml", ns).iterator();
        HashMap<String, FileSystem> filesystemsMap = new HashMap<String, FileSystem>();
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        while (it.hasNext()) {
            Element e = it.next();
            String jobXml = e.getTextTrim();
            Path pathSpecified = new Path(jobXml);
            Path path = pathSpecified.isAbsolute() ? pathSpecified : new Path(appPath, jobXml);
            FileSystem fs;
            if (filesystemsMap.containsKey(path.toUri().getAuthority())) {
              fs = filesystemsMap.get(path.toUri().getAuthority());
            }
            else {
              if (path.toUri().getAuthority() != null) {
                fs = has.createFileSystem(context.getWorkflow().getUser(), path.toUri(),
                        has.createJobConf(path.toUri().getAuthority()));
              }
              else {
                fs = context.getAppFileSystem();
              }
              filesystemsMap.put(path.toUri().getAuthority(), fs);

View Full Code Here

    }


    Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
            throws ActionExecutorException {
        try {
            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
            XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType());
            XConfiguration.injectDefaults(actionDefaults, actionConf);


            has.checkSupportedFilesystem(appPath.toUri());


            // Set the Java Main Class for the Java action to give to the Java launcher
            setJavaMain(actionConf, actionXml);


            parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf);

View Full Code Here

            }
            else {
                LOG.debug("Submitting the job through Job Client for action " + action.getId());


                // setting up propagation of the delegation token.
                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
                Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(has
                        .getMRDelegationTokenRenewer(launcherJobConf));
                launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt);


                // insert credentials tokens to launcher job conf if needed
                if (needInjectCredentials()) {

View Full Code Here

     * @return FileSystem
     * @throws HadoopAccessorException
     */
    private FileSystem getFileSystemFor(Path path, Context context, XConfiguration fsConf) throws HadoopAccessorException {
        String user = context.getWorkflow().getUser();
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        JobConf conf = has.createJobConf(path.toUri().getAuthority());
        XConfiguration.copy(context.getProtoActionConf(), conf);
        if (fsConf != null) {
            XConfiguration.copy(fsConf, conf);
        }
        return has.createFileSystem(user, path.toUri(), conf);
    }

View Full Code Here

     * @param group
     * @return FileSystem
     * @throws HadoopAccessorException
     */
    private FileSystem getFileSystemFor(Path path, String user) throws HadoopAccessorException {
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        JobConf jobConf = has.createJobConf(path.toUri().getAuthority());
        return has.createFileSystem(user, path.toUri(), jobConf);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.oozie.service.HadoopAccessorService

org.apache.hadoop.conf.Configuration

org.apache.hadoop.io.Text

org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.security.UserGroupInformation

org.apache.oozie.action.hadoop.AuthHelper

org.apache.oozie.action.hadoop.FsActionExecutor

org.apache.oozie.action.hadoop.FsELFunctions

org.apache.oozie.action.hadoop.JavaActionExecutor

org.apache.oozie.action.hadoop.LauncherMapper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.