Package org.elasticsearch.hadoop.cfg

Examples of org.elasticsearch.hadoop.cfg.Settings

86
87
88
89
90
91
92
93
94
95
96
    // NB: save the table properties in a special place but nothing else; otherwise the settings might trip on each other
    private void init(TableDesc tableDesc, boolean read) {
        Configuration cfg = getConf();
        // NB: we can't just merge the table properties in, we need to save them per input/output otherwise clashes occur which confuse Hive

        Settings settings = HadoopSettingsManager.loadFrom(cfg);
        //settings.setProperty((read ? HiveConstants.INPUT_TBL_PROPERTIES : HiveConstants.OUTPUT_TBL_PROPERTIES), IOUtils.propsToString(tableDesc.getProperties()));
        if (read) {
            // no generic setting
        }
        else {
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
    private void lazyInitializeWrite() {
        if (writeInitialized) {
            return;
        }
        writeInitialized = true;
        Settings settings = HadoopSettingsManager.loadFrom(tableProperties);

        InitializationUtils.setValueWriterIfNotSet(settings, HiveValueWriter.class, log);
        InitializationUtils.setFieldExtractorIfNotSet(settings, HiveFieldExtractor.class, log);
        InitializationUtils.setBytesConverterIfNeeded(settings, HiveBytesConverter.class, log);
        this.command = BulkCommands.create(settings);
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    }

    public EsHiveRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) {
        // force the table properties to be merged into the configuration
        // NB: the properties are also available in HiveConstants#OUTPUT_TBL_PROPERTIES
        Settings settings = SettingsManager.loadFrom(jc).merge(tableProperties);

        Log log = LogFactory.getLog(getClass());

        // NB: ESSerDe is already initialized at this stage but should still have a reference to the same cfg object
        // NB: the value writer is not needed by Hive but it's set for consistency and debugging purposes

        InitializationUtils.setValueWriterIfNotSet(settings, HiveValueWriter.class, log);
        InitializationUtils.setBytesConverterIfNeeded(settings, HiveBytesConverter.class, log);
        // set write resource
        settings.setResourceWrite(settings.getResourceWrite());

        HiveUtils.init(settings, log);

        return new EsHiveRecordWriter(jc, progress);
    }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
    public void setStoreLocation(String location, Job job) throws IOException {
        init(location, job, false);
    }

    private void init(String location, Job job, boolean read) {
        Settings settings = SettingsManager.loadFrom(job.getConfiguration()).merge(properties);

        settings = (read ? settings.setResourceRead(location) : settings.setResourceWrite(location));

        boolean changed = false;
        InitializationUtils.checkIdForOperation(settings);

        changed |= InitializationUtils.setValueWriterIfNotSet(settings, PigValueWriter.class, log);
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    public void setLocation(String location, Job job) throws IOException {
        init(location, job, true);

        Configuration cfg = job.getConfiguration();

        Settings settings = SettingsManager.loadFrom(cfg);
        IS_ES_10 = SettingsUtils.isEs10(settings);

        if (settings.getScrollFields() != null) {
            return;
        }

        extractProjection(cfg);
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    }

    @Override
    public FileSplit[] getSplits(JobConf job, int numSplits) throws IOException {
        // first, merge input table properties (since there's no access to them ...)
        Settings settings = SettingsManager.loadFrom(job);
        //settings.merge(IOUtils.propsFromString(settings.getProperty(HiveConstants.INPUT_TBL_PROPERTIES)));

        Log log = LogFactory.getLog(getClass());
        // move on to initialization
        InitializationUtils.setValueReaderIfNotSet(settings, HiveValueReader.class, log);
        settings.setProperty(InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS, StringUtils.concatenate(HiveUtils.columnToAlias(settings), ","));
        // set read resource
        settings.setResourceRead(settings.getResourceRead());
        HiveUtils.init(settings, log);

        // decorate original splits as FileSplit
        InputSplit[] shardSplits = super.getSplits(job, numSplits);
        FileSplit[] wrappers = new FileSplit[shardSplits.length];
View Full Code Here
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
        init(cfg);
    }

    // NB: all changes to the config objects are discarded before the job is submitted if _the old MR api_ is used
    private void init(Configuration cfg) throws IOException {
        Settings settings = SettingsManager.loadFrom(cfg);
        Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE));

        // lazy-init
        RestRepository client = null;

        InitializationUtils.checkIdForOperation(settings);
        InitializationUtils.checkIndexExistence(settings, client);

        if (HadoopCfgUtils.getReduceTasks(cfg) != null) {
            if (HadoopCfgUtils.getSpeculativeReduce(cfg)) {
                log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption");
            }
        }
        else {
            if (HadoopCfgUtils.getSpeculativeMap(cfg)) {
                log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption");
            }
        }

        Version.logVersion();
        log.info(String.format("Writing to [%s]", settings.getResourceWrite()));

        //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource()));
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    private static final String MAPPING_NAMES = "es.mapping.names";
    private static final boolean CASCADING_22_AVAILABLE = ObjectUtils.isClassPresent("cascading.tuple.type.CoercibleType", Tap.class.getClassLoader());

    static Settings addDefaultsToSettings(Properties flowProperties, Properties tapProperties, Log log) {
        Settings settings = SettingsManager.loadFrom(CascadingUtils.extractOriginalProperties(flowProperties)).merge(tapProperties);

        try {
            InitializationUtils.discoverNodesIfNeeded(settings, log);
            InitializationUtils.discoverEsVersion(settings, log);
        } catch (IOException ex) {
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    @Override
    public TupleEntryIterator openForRead(FlowProcess<Properties> flowProcess, ScrollQuery input) throws IOException {
        if (input == null) {
            // get original copy
            Settings settings = CascadingUtils.addDefaultsToSettings(CascadingUtils.extractOriginalProperties(flowProcess.getConfigCopy()), tapProperties, log);

            // will be closed by the query is finished
            RestRepository client = new RestRepository(settings);
            Field mapping = client.getMapping();
            Collection<String> fields = CascadingUtils.fieldToAlias(settings, getSourceFields());

            // validate if possible
            FieldPresenceValidation validation = settings.getFieldExistanceValidation();
            if (validation.isRequired()) {
                MappingUtils.validateMapping(fields, mapping, validation, log);
            }

            input = QueryBuilder.query(settings).fields(StringUtils.concatenate(fields,  ",")).build(client, new ScrollReader(new JdkValueReader(), mapping));
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
    // NB: save the table properties in a special place but nothing else; otherwise the settings might trip on each other
    private void init(TableDesc tableDesc, boolean read) {
        Configuration cfg = getConf();
        // NB: we can't just merge the table properties in, we need to save them per input/output otherwise clashes occur which confuse Hive

        Settings settings = SettingsManager.loadFrom(cfg);
        //settings.setProperty((read ? HiveConstants.INPUT_TBL_PROPERTIES : HiveConstants.OUTPUT_TBL_PROPERTIES), IOUtils.propsToString(tableDesc.getProperties()));
        if (read) {
            // no generic setting
        }
        else {
View Full Code Here
TOP

Related Classes of org.elasticsearch.hadoop.cfg.Settings

  • org.elasticsearch.hadoop.cascading.CascadingUtils
  • org.elasticsearch.hadoop.cascading.EsHadoopScheme
  • org.elasticsearch.hadoop.cascading.EsLocalScheme
  • org.elasticsearch.hadoop.cascading.EsLocalTap
  • org.elasticsearch.hadoop.hive.EsHiveInputFormat
  • org.elasticsearch.hadoop.hive.EsHiveOutputFormat
  • org.elasticsearch.hadoop.hive.EsSerDe
  • org.elasticsearch.hadoop.hive.EsStorageHandler
  • org.elasticsearch.hadoop.integration.rest.AbstractRestQueryTest
  • org.elasticsearch.hadoop.mr.EsInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.