Package org.elasticsearch.hadoop.cfg

Examples of org.elasticsearch.hadoop.cfg.Settings

89
90
91
92
93
94
95
96
97
98
99
100
        assertEquals(result, ba.toString());
    }

    @Test
    public void testParent() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_PARENT, "<5>");

        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_parent\":\"5\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
        assertEquals(result, ba.toString());
    }

    @Test
    public void testVersion() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_VERSION, "<3>");

        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_version\":\"3\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
        assertEquals(result, ba.toString());
    }

    @Test
    public void testTtl() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_TTL, "<2>");

        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_ttl\":\"2\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
        assertEquals(result, ba.toString());
    }

    @Test
    public void testTimestamp() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_TIMESTAMP, "<3>");
        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_timestamp\":\"3\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
        assertEquals(result, ba.toString());
    }

    @Test
    public void testRouting() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_ROUTING, "<4>");

        create(settings).write(map).copyTo(ba);
        String result = prefix() + "\"_routing\":\"4\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    }


    @Test
    public void testAll() throws Exception {
        Settings settings = settings();
        settings.setProperty(ConfigurationOptions.ES_MAPPING_ID, "n");
        settings.setProperty(ConfigurationOptions.ES_MAPPING_TTL, "<2>");
        settings.setProperty(ConfigurationOptions.ES_MAPPING_ROUTING, "s");

        create(settings).write(map).copyTo(ba);
        String result = "{\"" + operation + "\":{\"_id\":\"1\",\"_routing\":\"v\",\"_ttl\":\"2\"}}" + map();
        assertEquals(result, ba.toString());
    }
View Full Code Here
152
153
154
155
156
157
158
159
160
    }

    @Test(expected = EsHadoopIllegalArgumentException.class)
    public void testIdMandatory() throws Exception {
        assumeTrue(ConfigurationOptions.ES_OPERATION_UPDATE.equals(operation));
        Settings set = settings();
        set.setProperty(ConfigurationOptions.ES_MAPPING_ID, "");
        create(set).write(map).copyTo(ba);
    }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
        settings.setProperty(ConfigurationOptions.ES_WRITE_OPERATION, operation);
        return BulkCommands.create(settings);
    }

    private Settings settings() {
        Settings set = new TestSettings();
        InitializationUtils.setValueWriterIfNotSet(set, JdkValueWriter.class, null);
        InitializationUtils.setFieldExtractorIfNotSet(set, MapFieldExtractor.class, null);
        set.setResourceWrite("foo/bar");
        if (isUpdateOp()) {
            set.setProperty(ConfigurationOptions.ES_MAPPING_ID, "<2>");
        }
        return set;
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    }

    public EsHiveRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) {
        // force the table properties to be merged into the configuration
        // NB: the properties are also available in HiveConstants#OUTPUT_TBL_PROPERTIES
        Settings settings = HadoopSettingsManager.loadFrom(jc).merge(tableProperties);

        Log log = LogFactory.getLog(getClass());

        // NB: ESSerDe is already initialized at this stage but should still have a reference to the same cfg object
        // NB: the value writer is not needed by Hive but it's set for consistency and debugging purposes

        InitializationUtils.setValueWriterIfNotSet(settings, HiveValueWriter.class, log);
        InitializationUtils.setBytesConverterIfNeeded(settings, HiveBytesConverter.class, log);
        // set write resource
        settings.setResourceWrite(settings.getResourceWrite());

        HiveUtils.init(settings, log);

        return new EsHiveRecordWriter(jc, progress);
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    }

    @Override
    public FileSplit[] getSplits(JobConf job, int numSplits) throws IOException {
        // first, merge input table properties (since there's no access to them ...)
        Settings settings = HadoopSettingsManager.loadFrom(job);
        //settings.merge(IOUtils.propsFromString(settings.getProperty(HiveConstants.INPUT_TBL_PROPERTIES)));

        Log log = LogFactory.getLog(getClass());
        // move on to initialization
        InitializationUtils.setValueReaderIfNotSet(settings, HiveValueReader.class, log);
        settings.setProperty(InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS, StringUtils.concatenateAndUriEncode(HiveUtils.columnToAlias(settings), ","));
        // set read resource
        settings.setResourceRead(settings.getResourceRead());
        HiveUtils.init(settings, log);

        // decorate original splits as FileSplit
        InputSplit[] shardSplits = super.getSplits(job, numSplits);
        FileSplit[] wrappers = new FileSplit[shardSplits.length];
View Full Code Here
TOP

Related Classes of org.elasticsearch.hadoop.cfg.Settings

  • org.elasticsearch.hadoop.cascading.CascadingUtils
  • org.elasticsearch.hadoop.cascading.EsHadoopScheme
  • org.elasticsearch.hadoop.cascading.EsLocalScheme
  • org.elasticsearch.hadoop.cascading.EsLocalTap
  • org.elasticsearch.hadoop.hive.EsHiveInputFormat
  • org.elasticsearch.hadoop.hive.EsHiveOutputFormat
  • org.elasticsearch.hadoop.hive.EsSerDe
  • org.elasticsearch.hadoop.hive.EsStorageHandler
  • org.elasticsearch.hadoop.integration.rest.AbstractRestQueryTest
  • org.elasticsearch.hadoop.mr.EsInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.