Package org.elasticsearch.hadoop.cfg

Examples of org.elasticsearch.hadoop.cfg.Settings


    private static final String MAPPING_NAMES = "es.mapping.names";
    private static final boolean CASCADING_22_AVAILABLE = ObjectUtils.isClassPresent("cascading.tuple.type.CoercibleType", Tap.class.getClassLoader());

    static Settings addDefaultsToSettings(Properties flowProperties, Properties tapProperties, Log log) {
        Settings settings = HadoopSettingsManager.loadFrom(CascadingUtils.extractOriginalProperties(flowProperties)).merge(tapProperties);

        InitializationUtils.discoverNodesIfNeeded(settings, log);
        InitializationUtils.discoverEsVersion(settings, log);

        InitializationUtils.setValueWriterIfNotSet(settings, CascadingValueWriter.class, log);
View Full Code Here


        init(cfg);
    }

    // NB: all changes to the config objects are discarded before the job is submitted if _the old MR api_ is used
    private void init(Configuration cfg) throws IOException {
        Settings settings = HadoopSettingsManager.loadFrom(cfg);
        Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE));

        // lazy-init
        RestRepository client = null;

        InitializationUtils.checkIdForOperation(settings);
        InitializationUtils.checkIndexExistence(settings, client);

        if (HadoopCfgUtils.getReduceTasks(cfg) != null) {
            if (HadoopCfgUtils.getSpeculativeReduce(cfg)) {
                log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption");
            }
        }
        else {
            if (HadoopCfgUtils.getSpeculativeMap(cfg)) {
                log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption");
            }
        }

        Version.logVersion();
        log.info(String.format("Writing to [%s]", settings.getResourceWrite()));

        //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource()));
    }
View Full Code Here

            if (log.isTraceEnabled()) {
                log.trace(String.format("EsRecordWriter instance [%s] initiating discovery of target shard...",
                        currentInstance));
            }

            Settings settings = HadoopSettingsManager.loadFrom(cfg).copy();

            if (log.isTraceEnabled()) {
                log.trace(String.format("Init shard writer from cfg %s", HadoopCfgUtils.asProperties(cfg)));
            }

            InitializationUtils.setValueWriterIfNotSet(settings, WritableValueWriter.class, log);
            InitializationUtils.setBytesConverterIfNeeded(settings, WritableBytesConverter.class, log);
            InitializationUtils.setFieldExtractorIfNotSet(settings, MapWritableFieldExtractor.class, log);

            PartitionWriter pw = RestService.createWriter(settings, currentInstance, -1, log);

            this.repository = pw.repository;
           
            this.beat = new HeartBeat(progressable, cfg, settings.getHeartBeatLead(), log);
            this.beat.start();
        }
View Full Code Here

        assertNotNull(shards);
    }

    @Test
    public void testQueryBuilder() throws Exception {
        Settings sets = settings.copy();
        sets.setProperty(ConfigurationOptions.ES_QUERY, "?q=me*");
        QueryBuilder qb = QueryBuilder.query(sets);
        Field mapping = client.getMapping();
        ScrollReader reader = new ScrollReader(new JdkValueReader(), mapping, true, "_metadata");

        int count = 0;
View Full Code Here

        Map<Shard, Node> targetShards = client.getReadTargetShards();

        Field mapping = client.getMapping();
        ScrollReader reader = new ScrollReader(new JdkValueReader(), mapping, true, "_metadata");

        Settings sets = settings.copy();
        sets.setProperty(ConfigurationOptions.ES_QUERY, "?q=me*");

        String nodeId = targetShards.values().iterator().next().getId();
        ScrollQuery query = QueryBuilder.query(sets)
                .shard("0")
                .onlyNode(nodeId)
View Full Code Here

        Resource res = createResource("foo/bar/_search=?somequery&bla=bla");
        assertEquals("foo/bar", res.indexAndType());
    }

    private Resource createResource(String target) {
        Settings s = new TestSettings();
        s.setProperty(ConfigurationOptions.ES_RESOURCE, target);
        return new Resource(s, true);
    }
View Full Code Here

        return false;
    }

    public static <T> void saveSchemaIfNeeded(Object conf, ValueWriter<T> schemaWriter, T schema, Log log) {
        Settings settings = HadoopSettingsManager.loadFrom(conf);

        if (settings.getIndexAutoCreate()) {
            RestRepository client = new RestRepository(settings);
            if (!client.indexExists(false)) {
                if (schemaWriter == null) {
                    log.warn(String.format("No mapping found [%s] and no schema found; letting Elasticsearch perform auto-mapping...",  settings.getResourceWrite()));
                }
                else {
                    log.info(String.format("No mapping found [%s], creating one based on given schema", settings.getResourceWrite()));
                    ContentBuilder builder = ContentBuilder.generate(schemaWriter).value(schema).flush();
                    BytesArray content = ((FastByteArrayOutputStream) builder.content()).bytes();
                    builder.close();
                    client.putMapping(content);
                    if (log.isDebugEnabled()) {
View Full Code Here

    public void setStoreLocation(String location, Job job) throws IOException {
        init(location, job, false);
    }

    private void init(String location, Job job, boolean read) {
        Settings settings = HadoopSettingsManager.loadFrom(job.getConfiguration()).merge(properties);

        settings = (read ? settings.setResourceRead(location) : settings.setResourceWrite(location));

        boolean changed = false;
        InitializationUtils.checkIdForOperation(settings);

        changed |= InitializationUtils.setValueWriterIfNotSet(settings, PigValueWriter.class, log);
View Full Code Here

    public void setLocation(String location, Job job) throws IOException {
        init(location, job, true);

        Configuration cfg = job.getConfiguration();

        Settings settings = HadoopSettingsManager.loadFrom(cfg);
        IS_ES_10 = SettingsUtils.isEs10(settings);

        if (settings.getScrollFields() != null) {
            return;
        }

        extractProjection(cfg);
    }
View Full Code Here

        assertEquals(result, ba.toString());
    }

    @Test
    public void testConstantId() throws Exception {
        Settings settings = settings();
        noId = true;
        settings.setProperty(ConfigurationOptions.ES_MAPPING_ID, "<1>");

        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_id\":\"1\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here

TOP

Related Classes of org.elasticsearch.hadoop.cfg.Settings

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.