Package org.apache.oozie.util

Examples of org.apache.oozie.util.XConfiguration

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        }
        // generate the 'runConf' for this action
        // Step 1: runConf = createdConf
        Configuration runConf = null;
        try {
            runConf = new XConfiguration(new StringReader(createdConf));
        }
        catch (IOException e1) {
            log.warn("Configuration parse error in:" + createdConf);
            throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
        }
        // Step 2: Merge local properties into runConf
        // extract 'property' tags under 'configuration' block in the
        // coordinator.xml (saved in actionxml column)
        // convert Element to XConfiguration
        Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace())
                .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
                                                                                  workflowProperties.getNamespace());
        if (configElement != null) {
            String strConfig = XmlUtils.prettyPrint(configElement).toString();
            Configuration localConf;
            try {
                localConf = new XConfiguration(new StringReader(strConfig));
            }
            catch (IOException e1) {
                log.warn("Configuration parse error in:" + strConfig);
                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
            }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
            // log.debug("%%% merged runconf=" +
            // XmlUtils.prettyPrint(runConf).toString());
            DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
            try {
                boolean startJob = true;
                Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
                SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED,
                                                SlaAppType.COORDINATOR_ACTION, log);

                // Normalize workflow appPath here;
                JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
                String wfId = dagEngine.submitJob(conf, startJob);
                coordAction.setStatus(CoordinatorAction.Status.RUNNING);
                coordAction.setExternalId(wfId);
                coordAction.incrementAndGetPending();
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
        InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
        WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
        try {
            XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
            WorkflowApp app = wps.parseDef(conf, authToken);
            XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, true);
            WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();

            String user = conf.get(OozieClient.USER_NAME);
            String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
            URI uri = new URI(conf.get(OozieClient.APP_PATH));
            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
            Configuration fsConf = has.createJobConf(uri.getAuthority());
            FileSystem fs = has.createFileSystem(user, uri, fsConf);

            Path configDefault = null;
            // app path could be a directory
            Path path = new Path(uri.getPath());
            if (!fs.isFile(path)) {
                configDefault = new Path(path, CONFIG_DEFAULT);
            } else {
                configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
            }

            if (fs.exists(configDefault)) {
                try {
                    Configuration defaultConf = new XConfiguration(fs.open(configDefault));
                    PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                    XConfiguration.injectDefaults(defaultConf, conf);
                }
                catch (IOException ex) {
                    throw new IOException("default configuration file, " + ex.getMessage(), ex);
                }
            }

            PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);

            // Resolving all variables in the job properties.
            // This ensures the Hadoop Configuration semantics is preserved.
            XConfiguration resolvedVarsConf = new XConfiguration();
            for (Map.Entry<String, String> entry : conf) {
                resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
            }
            conf = resolvedVarsConf;

            WorkflowInstance wfInstance;
            try {
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
        }

        public Configuration getProtoActionConf() {
            String s = workflow.getProtoActionConf();
            try {
                return new XConfiguration(new StringReader(s));
            }
            catch (IOException ex) {
                throw new RuntimeException(ex);
            }
        }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    public void testKill() throws Exception {
        _testAction(RestConstants.JOB_ACTION_KILL, null);
    }

    public void testReRun() throws Exception {
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.USER_NAME, getTestUser());
        Path appPath = new Path(getFsTestCaseDir(), "app");
        getFileSystem().mkdirs(appPath);
        getFileSystem().create(new Path(appPath, "workflow.xml")).close();
        conf.set(OozieClient.APP_PATH, appPath.toString());

        _testAction(RestConstants.JOB_ACTION_RERUN, conf);
    }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
        _testAction(RestConstants.JOB_ACTION_RERUN, conf);
    }

    public void testInvalidReRunConfigurations() throws Exception {
        Configuration conf = new XConfiguration();
        Path appPath = new Path(getFsTestCaseDir(), "app");
        getFileSystem().mkdirs(appPath);
        getFileSystem().create(new Path(appPath, "workflow.xml")).close();
        conf.set(OozieClient.APP_PATH, appPath.toString());
        _testAction(RestConstants.JOB_ACTION_RERUN, conf);
    }
View Full Code Here
441
442
443
444
445
446
447
448
449
450
            StringBuilder actionXml = new StringBuilder(action);
            StringBuilder existList = new StringBuilder();
            StringBuilder nonExistList = new StringBuilder();
            StringBuilder nonResolvedList = new StringBuilder();
            getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList);
            Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
            coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf);
            return actionXml.toString();
        }
    }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
        IOUtils.copyStream(is, os);

        Path appSoPath = new Path("lib/test.so");
        getFileSystem().create(new Path(getAppPath(), appSoPath)).close();

        XConfiguration protoConf = new XConfiguration();
        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
        protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, appJarPath.toString(), appSoPath.toString());


        WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
        WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
        action.setType(ae.getType());
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
        os = fs.create(new Path(dir, "b"));
        arr = new byte[2];
        os.write(arr);
        os.close();

        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "appPath");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set("test.dir", getTestCaseDir());
        conf.set("file1", file1);
        conf.set("file2", file2);
        conf.set("file3", "${file2}");
        conf.set("dir", dir);

        LiteWorkflowApp def =
                new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef("end")).addNode(new EndNodeDef("end"));
        LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
            Path script = new Path(getTestCaseDir(), "script.q");
            Writer w = new FileWriter(script.toString());
            w.write(getHiveScript("${IN}", "${OUT}"));
            w.close();

            XConfiguration jobConf = new XConfiguration();
            jobConf.set("mapreduce.framework.name", "yarn");

            jobConf.set("oozie.hive.log.level", "DEBUG");

            jobConf.set("user.name", getTestUser());
            jobConf.set("group.name", getTestGroup());
            jobConf.setInt("mapred.map.tasks", 1);
            jobConf.setInt("mapred.map.max.attempts", 1);
            jobConf.setInt("mapred.reduce.max.attempts", 1);
            jobConf.set("mapred.job.tracker", getJobTrackerUri());
            jobConf.set("fs.default.name", getNameNodeUri());
            jobConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:" + getTestCaseDir() + "/db;create=true");
            jobConf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver");
            jobConf.set("javax.jdo.option.ConnectionUserName", "sa");
            jobConf.set("javax.jdo.option.ConnectionPassword", " ");

            SharelibUtils.addToDistributedCache("hive", fs, getFsTestCaseDir(), jobConf);

            HiveMain.setHiveScript(jobConf, script.toString(), new String[]{"IN=" + inputDir.toUri().getPath(),
                    "OUT=" + outputDir.toUri().getPath()});

            File actionXml = new File(getTestCaseDir(), "action.xml");
            OutputStream os = new FileOutputStream(actionXml);
            jobConf.writeXml(os);
            os.close();

            //needed in the testcase classpath
            URL url = Thread.currentThread().getContextClassLoader().getResource("PigMain.txt");
            File classPathDir = new File(url.getPath()).getParentFile();
            assertTrue(classPathDir.exists());
            File hiveSite = new File(classPathDir, "hive-site.xml");

            InputStream is = IOUtils.getResourceAsStream("user-hive-default.xml", -1);
            os = new FileOutputStream(new File(classPathDir, "hive-default.xml"));
            IOUtils.copyStream(is, os);

            File outputDataFile = new File(getTestCaseDir(), "outputdata.properties");

            setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis());
            setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
            setSystemProperty("oozie.action.output.properties", outputDataFile.getAbsolutePath());

            new LauncherSecurityManager();
            String user = System.getProperty("user.name");
            try {
                os = new FileOutputStream(hiveSite);
                jobConf.writeXml(os);
                os.close();
                HiveMain.main(null);
            }
            catch (SecurityException ex) {
                if (LauncherSecurityManager.getExitInvoked()) {
View Full Code Here
TOP

Related Classes of org.apache.oozie.util.XConfiguration

  • org.apache.oozie.action.hadoop.ActionExecutorTestCase
  • org.apache.oozie.action.hadoop.ActionExecutorTestCase$Context
  • org.apache.oozie.action.hadoop.FsActionExecutor
  • org.apache.oozie.action.hadoop.HiveActionExecutor
  • org.apache.oozie.action.hadoop.JavaActionExecutor
  • org.apache.oozie.action.hadoop.MapReduceActionExecutor
  • org.apache.oozie.action.hadoop.SqoopActionExecutor
  • org.apache.oozie.action.hadoop.TestDistCpActionExecutor
  • org.apache.oozie.action.hadoop.TestFsActionExecutor
  • org.apache.oozie.action.hadoop.TestFsELFunctions
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.