Examples of azkaban.common.utils.Props

azkaban.common.utils.Props
A less fucked properties class - Implements Map instead of extending HashMap - Hash helpers for getting typed values - Support for getting with an automatic default value - Throws exception if property is not defined @author jay

            }


        }


        public void configure(JobConf job) {
            Props props = HadoopUtils.getPropsFromJob(job);


            try {
                _position = 0;
                _conf = job;


                _taskId = job.get("mapred.task.id");


                storeName = props.get("voldemort.store.name");
                taskIndexFileName = new Path(FileOutputFormat.getOutputPath(_conf),
                                             _conf.get("voldemort.index.filename") + "_" + _taskId);
                taskValueFileName = new Path(FileOutputFormat.getOutputPath(_conf),
                                             _conf.get("voldemort.data.filename") + "_" + _taskId);

View Full Code Here

                                           e);
            }
        }


        public void configure(JobConf conf) {
            Props props = HadoopUtils.getPropsFromJob(conf);


            // get the voldemort cluster.xml and store.xml files.
            try {
                _cluster = HadoopUtils.readCluster(props.get("voldemort.cluster.file"), conf);
                _storeDef = HadoopUtils.readStoreDef(props.get("voldemort.store.file"),
                                                     props.get("voldemort.store.name"),
                                                     conf);


                _checkPercent = conf.getInt("input.data.check.percent", 0);
                _routingStrategy = new ConsistentRoutingStrategy(_cluster,
                                                                 _storeDef.getReplicationFactor());

View Full Code Here


        config.addResource(new Path(new File(hadoopConfDir, "hadoop-default.xml").getAbsolutePath()));
        config.addResource(new Path(new File(hadoopConfDir, "hadoop-site.xml").getAbsolutePath()));


        // copy to props
        Props props = new Props(parent);
        for(Entry<String, String> entry: config)
            props.put(entry.getKey(), config.get(entry.getKey()));


        return props;
    }

View Full Code Here

            throw new UndefinedPropertyException("The required property azkaban.props was not found in the Configuration.");
        try {
            ByteArrayInputStream input = new ByteArrayInputStream(propsString.getBytes("UTF-8"));
            Properties properties = new Properties();
            properties.load(input);
            return new Props(null, properties);
        } catch(IOException e) {
            throw new RuntimeException("This is not possible!", e);
        }
    }

View Full Code Here

        if(fs.exists(path)) {
            InputStream input = fs.open(path);
            try {
                // wrap it up in another layer so that the user can override
                // properties
                Props p = new Props(null, input);
                return new Props(p);
            } finally {
                input.close();
            }
        } else {
            return new Props();
        }
    }

View Full Code Here

    }


    @Override
    public void configure(JobConf conf) {
        super.configure(conf);
        Props props = HadoopUtils.getPropsFromJob(conf);


        _keySelection = props.getString("key.selection", null);
        _valSelection = props.getString("value.selection", null);
        _inputKeySerializer = getSchemaFromJob(conf, "mapper.input.key.schema");
        _inputValueSerializer = getSchemaFromJob(conf, "mapper.input.value.schema");
        String _keyTransClass = props.getString("key.transformation.class", null);
        String _valueTransClass = props.getString("value.transformation.class", null);


        if(_keyTransClass != null)
            _keyTrans = (StoreBuilderTransformation) Utils.callConstructor(_keyTransClass);
        if(_valueTransClass != null)
            _valTrans = (StoreBuilderTransformation) Utils.callConstructor(_valueTransClass);

View Full Code Here

        valueCompressor = new CompressionStrategyFactory().get(valueSerializerDefinition.getCompression());


        routingStrategy = new ConsistentRoutingStrategy(getCluster(),
                                                        getStoreDef().getReplicationFactor());


        Props props = HadoopUtils.getPropsFromJob(conf);


    }

View Full Code Here

        this._jobDirs = jobDirs;
        this._baseClassLoader = classLoader;
    }


    public Job loadJob(String jobName, boolean ignoreDep) {
        return loadJob(jobName, new Props(), ignoreDep);
    }

View Full Code Here

                } catch (Exception e) {
                    logger.info("Ignoring unknown directory found in logs:" + execDir.getAbsolutePath(), e);
                    continue;
                }
                if(runProps.canRead()) {
                    Props props = new Props(null, runProps.getAbsolutePath());
                    start = new DateTime(props.getLong("start"));
                    end = new DateTime(props.getLong("end"));
                    succeeded = props.getBoolean("succeeded");
                }
                String logFile = jobName + File.separator + execDir.getName() + File.separator
                                 + jobName + "." + execDir.getName() + ".log";
                execs.add(new JobExecution(jobName,
                                           start == null ? dirDate : start,

View Full Code Here

                                              propsOverrides,
                                              pathOverrides,
                                              _baseClassLoader);


        if(pathOverrides != null) {
            Props dirProps = loadLocalNonJobProps(jobDir, _defaultProps);
            for(Map.Entry<File, File> override: pathOverrides.entrySet()) {
                loadJobDescriptorsWithoutDependencies(m,
                                                      jobDir,
                                                      override.getValue(),
                                                      dirProps,

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of azkaban.common.utils.Props

azkaban.app.AzkabanApplication

azkaban.app.JobManager

azkaban.app.LoggingJob

azkaban.app.PropsUtils

azkaban.app.PropUtilsTest

azkaban.common.jobs.AbstractJob

azkaban.flow.ComposedExecutableFlow

azkaban.flow.ComposedExecutableFlow$DependeeCallback

azkaban.flow.ComposedExecutableFlowTest

azkaban.flow.GroupedExecutableFlow

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.